From mdounin at mdounin.ru Sun Jan 1 17:35:30 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sun, 1 Jan 2023 20:35:30 +0300 Subject: [PATCH] Dynamic rate limiting for limit_req module In-Reply-To: References: Message-ID: Hello! On Fri, Dec 30, 2022 at 10:23:12PM +0000, J Carter wrote: > Please find below a patch to enable dynamic rate limiting for > limit_req module. Thanks for the patch, and Happy New Year. The fundamental problem with dynamic rates in limit_req, which is a leaky bucket limiter, is that the rate is a property which applies to multiple requests as accumulated in the bucket (and this is basically why it is configured in the limit_req_zone, and not in limit_req), while the dynamically calculated rate is a property of the last request. This can easily result in counter-intuitive behaviour. For example, consider 5 requests with 1 second interval, assuming burst 2, and rate being 1r/m or 100r/s depending on some request properties: - 1st request, rate 1r/m, request is accepted - 2st request, rate 1r/m, request is accepted - 3rd request, rate 1r/m, request is rejected, since there is no room in the bucket - 4th request, rate 100r/s, request is accepted - 5th request, rate 1r/m, request is accepted (unexpectedly) Note that the 5th request is accepted, while it is mostly equivalent to the 3rd request, and one could expect it to be rejected. But it happens to to be accepted because the 4th request "cleared" the bucket. Could you please clarify how this problem is addressed in your patch? Note well that the same limits now can be configured separately, in distinct shared memory zones. This makes it possible to limit them requests consistently, without any unexpected deviations from the configured behaviour if requests with different rates interfere. > /* ----------------------------EXAMPLE---------------------------------*/ > > geo $traffic_tier { > default free; > 127.0.1.0/24 basic; > 127.0.2.0/24 premium; > } > > map $traffic_tier $rate { > free 1r/m; > basic 2r/m; > premium 1r/s; > } > > limit_req_zone $binary_remote_addr zone=one:10m rate=$rate; > > server { > limit_req zone=one; > listen 80; > server_name localhost; > return 200; > } > > curl --interface 127.0.X.X localhost Just in case, the very same behaviour can be implemented with multiple limit_req_zones, with something like this: map $traffic_tier $limit_free { free $binary_remote_addr; } map $traffic_tier $limit_basic { basic $binary_remote_addr; } map $traffic_tier $limit_premium { premium $binary_remote_addr; } limit_req_zone $limit_free zone=free:10m rate=1r/m; limit_req_zone $limit_basic zone=basic:10m rate=2r/m; limit_req_zone $limit_premium zone=premium:10m rate=1r/s; limit_req zone=free; limit_req zone=basic; limit_req zone=premium; >From the example it is not very clear how the suggested change is beneficial. See below for some comments about the code. [...] > # HG changeset patch > # User jordanc.carter at outlook.com > # Date 1672437935 0 > # Fri Dec 30 22:05:35 2022 +0000 > # Branch dynamic-rate-limiting > # Node ID b2bd50efa81e5aeeb9b8f84ee0af34463add07fa > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > Changed 'rate=' to complex value and added limits to the rate value to prevent integer overflow/underflow > > diff -r 07b0bee87f32 -r b2bd50efa81e src/http/modules/ngx_http_limit_req_module.c > --- a/src/http/modules/ngx_http_limit_req_module.c Wed Dec 21 14:53:27 2022 +0300 > +++ b/src/http/modules/ngx_http_limit_req_module.c Fri Dec 30 22:05:35 2022 +0000 > @@ -26,6 +26,7 @@ > /* integer value, 1 corresponds to 0.001 r/s */ > ngx_uint_t excess; > ngx_uint_t count; > + ngx_uint_t rate; > u_char data[1]; > } ngx_http_limit_req_node_t; > > @@ -41,7 +42,7 @@ > ngx_http_limit_req_shctx_t *sh; > ngx_slab_pool_t *shpool; > /* integer value, 1 corresponds to 0.001 r/s */ > - ngx_uint_t rate; > + ngx_http_complex_value_t rate; > ngx_http_complex_value_t key; > ngx_http_limit_req_node_t *node; > } ngx_http_limit_req_ctx_t; > @@ -66,9 +67,9 @@ > > static void ngx_http_limit_req_delay(ngx_http_request_t *r); > static ngx_int_t ngx_http_limit_req_lookup(ngx_http_limit_req_limit_t *limit, > - ngx_uint_t hash, ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account); > + ngx_uint_t hash, ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account, ngx_uint_t rate); > static ngx_msec_t ngx_http_limit_req_account(ngx_http_limit_req_limit_t *limits, > - ngx_uint_t n, ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit); > + ngx_uint_t n, ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit, ngx_uint_t rate); > static void ngx_http_limit_req_unlock(ngx_http_limit_req_limit_t *limits, > ngx_uint_t n); > static void ngx_http_limit_req_expire(ngx_http_limit_req_ctx_t *ctx, > @@ -195,10 +196,13 @@ > ngx_http_limit_req_handler(ngx_http_request_t *r) > { > uint32_t hash; > - ngx_str_t key; > + ngx_str_t key, rate_s; > ngx_int_t rc; > ngx_uint_t n, excess; > + ngx_uint_t scale; > + ngx_uint_t rate; > ngx_msec_t delay; > + u_char *p; > ngx_http_limit_req_ctx_t *ctx; > ngx_http_limit_req_conf_t *lrcf; > ngx_http_limit_req_limit_t *limit, *limits; > @@ -243,10 +247,34 @@ > > hash = ngx_crc32_short(key.data, key.len); > > + if (ngx_http_complex_value(r, &ctx->rate, &rate_s) != NGX_OK) { > + ngx_http_limit_req_unlock(limits, n); > + return NGX_HTTP_INTERNAL_SERVER_ERROR; > + } > + > + scale = 1; > + rate = NGX_ERROR; > + > + if (rate_s.len > 8) { > + > + rate = (ngx_uint_t) ngx_atoi(rate_s.data + 5, rate_s.len - 8); > + > + p = rate_s.data + rate_s.len - 3; > + if (ngx_strncmp(p, "r/m", 3) == 0) { > + scale = 60; > + } else if (ngx_strncmp(p, "r/s", 3) != 0){ > + rate = NGX_ERROR; > + } > + } Note that this approach implies parsing rate on each request. This is something we usually try to avoid as long as a static string is configured in advance, see these commits for an example: changeset: 7503:b82162b8496a user: Ruslan Ermilov date: Wed Apr 24 16:38:51 2019 +0300 summary: Added ngx_http_set_complex_value_size_slot(). changeset: 7504:c19ca381b2e6 user: Ruslan Ermilov date: Wed Apr 24 16:38:54 2019 +0300 summary: Variables support in limit_rate and limit_rate_after (ticket #293). changeset: 7505:16a1adadf437 user: Ruslan Ermilov date: Wed Apr 24 16:38:56 2019 +0300 summary: Variables support in proxy_upload_rate and proxy_download_rate. A more self-contained solution can be seen in the image filter (src/http/modules/ngx_http_image_filter.c), which uses values parsed during configuration parsing if there are no variables, and complex values if there are variables (e.g., "imcf->angle" and "imcf->acv"). > + > + rate = (rate != 0 && rate < NGX_MAX_INT_T_VALUE / 60000000 - 1001) ? > + rate * 1000 / scale : > + NGX_MAX_INT_T_VALUE / 60000000 - 1001; > + > ngx_shmtx_lock(&ctx->shpool->mutex); > > rc = ngx_http_limit_req_lookup(limit, hash, &key, &excess, > - (n == lrcf->limits.nelts - 1)); > + (n == lrcf->limits.nelts - 1), rate); > > ngx_shmtx_unlock(&ctx->shpool->mutex); > > @@ -291,7 +319,7 @@ > excess = 0; > } > > - delay = ngx_http_limit_req_account(limits, n, &excess, &limit); > + delay = ngx_http_limit_req_account(limits, n, &excess, &limit, rate); > > if (!delay) { > r->main->limit_req_status = NGX_HTTP_LIMIT_REQ_PASSED; > @@ -403,7 +431,7 @@ > > static ngx_int_t > ngx_http_limit_req_lookup(ngx_http_limit_req_limit_t *limit, ngx_uint_t hash, > - ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account) > + ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account, ngx_uint_t rate) > { > size_t size; > ngx_int_t rc, excess; > @@ -412,7 +440,6 @@ > ngx_rbtree_node_t *node, *sentinel; > ngx_http_limit_req_ctx_t *ctx; > ngx_http_limit_req_node_t *lr; > - > now = ngx_current_msec; > > ctx = limit->shm_zone->data; Note that this is a style change, and it is wrong. > @@ -446,12 +473,14 @@ > > if (ms < -60000) { > ms = 1; > - The same here. > } else if (ms < 0) { > ms = 0; > + } else if (ms > 60000) { > + ms = 60000; > } This seems to be separate change, irrelevant to what the patch does. If at all, it should be submitted as a separate patch, see http://nginx.org/en/docs/contributing_changes.html. Note though that this changes looks wrong to me, as it will break request accounting on large time intervals with large burst, such as with rate=1r/m burst=100. Note well that it fails to use an empty line before "} else ", which is a style bug. > > - excess = lr->excess - ctx->rate * ms / 1000 + 1000; > + lr->rate = rate; > + excess = lr->excess - lr->rate * ms / 1000 + 1000; > > if (excess < 0) { > excess = 0; > @@ -510,6 +539,7 @@ > > lr->len = (u_short) key->len; > lr->excess = 0; > + lr->rate = rate; The "lr->rate", which is saved in the shared memory, seems to be never used except during processing of the same request. Any reasons to keep it in the shared memory, reducing the number of states which can be stored there? > > ngx_memcpy(lr->data, key->data, key->len); > > @@ -534,7 +564,7 @@ > > static ngx_msec_t > ngx_http_limit_req_account(ngx_http_limit_req_limit_t *limits, ngx_uint_t n, > - ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit) > + ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit, ngx_uint_t rate) > { > ngx_int_t excess; > ngx_msec_t now, delay, max_delay; > @@ -543,13 +573,13 @@ > ngx_http_limit_req_node_t *lr; > > excess = *ep; > + max_delay = 0; > Note that this is a style change, since the max_delay is set in all possible code paths, and there are no reasons to additionally initialize it. > if ((ngx_uint_t) excess <= (*limit)->delay) { > max_delay = 0; > > } else { > - ctx = (*limit)->shm_zone->data; > - max_delay = (excess - (*limit)->delay) * 1000 / ctx->rate; > + max_delay = (excess - (*limit)->delay) * 1000 / rate; > } > > while (n--) { > @@ -570,9 +600,11 @@ > > } else if (ms < 0) { > ms = 0; > + } else if (ms > 60000) { > + ms = 60000; > } See above. [...] -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Sun Jan 1 19:18:35 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Sun, 01 Jan 2023 22:18:35 +0300 Subject: [PATCH] Updated advisory links Message-ID: <165e9f39cfc8815cfc18.1672600715@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1672600531 -10800 # Sun Jan 01 22:15:31 2023 +0300 # Node ID 165e9f39cfc8815cfc18b7cf6cbf712059585608 # Parent 8033ffaedeb9f029897d464d85454b9cffa35fd9 Updated advisory links. Mailman 3 was unusable, and therefore list software was changed back to Mailman 2, with all the archives imported into pipermail. Advisory links updated accordingly. diff --git a/xml/en/security_advisories.xml b/xml/en/security_advisories.xml --- a/xml/en/security_advisories.xml +++ b/xml/en/security_advisories.xml @@ -26,7 +26,7 @@ Patches are signed using one of the @@ -35,7 +35,7 @@ Patches are signed using one of the From jordanc.carter at outlook.com Mon Jan 2 06:21:03 2023 From: jordanc.carter at outlook.com (J Carter) Date: Mon, 2 Jan 2023 06:21:03 +0000 Subject: [PATCH] Dynamic rate limiting for limit_req module In-Reply-To: References: Message-ID: Hello, Happy new year to you too, and thank you for the detailed feedback. > "The fundamental problem with dynamic rates in limit_req, which is > a leaky bucket limiter, is that the rate is a property which > applies to multiple requests as accumulated in the bucket (and > this is basically why it is configured in the limit_req_zone, and > not in limit_req), while the dynamically calculated rate is a > property of the last request. > > This can easily result in counter-intuitive behaviour. > For example, consider 5 requests with 1 second interval, assuming > burst 2, and rate being 1r/m or 100r/s depending on some request > properties: > > - 1st request, rate 1r/m, request is accepted > - 2st request, rate 1r/m, request is accepted > - 3rd request, rate 1r/m, request is rejected, since there is no > room in the bucket > - 4th request, rate 100r/s, request is accepted > - 5th request, rate 1r/m, request is accepted (unexpectedly) > > Note that the 5th request is accepted, while it is mostly > equivalent to the 3rd request, and one could expect it to be > rejected. But it happens to to be accepted because the 4th > request "cleared" the bucket." This is true - however I do wonder if the behavior (clearing the bucket) is truly counter intuitive in that scenario. Should it not be expected that this user/key/state that has been assigned 100r/s (even if just for one request) would have it's outstanding excess requests recalculated (and in this case absolved) by that rate increase? You have after all assigned it more capacity. I'm not sure how to elegantly avoid this if it is an issue, since there is no 'standard' rate to reference (it could interpolate over time/requests, but that might be even more confusing). > "Note well that the same limits now can be configured separately, > in distinct shared memory zones. This makes it possible to limit > them requests consistently, without any unexpected deviations from > the configured behaviour if requests with different rates > interfere. > > Just in case, the very same behaviour can be implemented with > multiple limit_req_zones, with something like this: > > map $traffic_tier $limit_free { > free $binary_remote_addr; > } > > map $traffic_tier $limit_basic { > basic $binary_remote_addr; > } > > map $traffic_tier $limit_premium { > premium $binary_remote_addr; > } > > limit_req_zone $limit_free zone=free:10m rate=1r/m; > limit_req_zone $limit_basic zone=basic:10m rate=2r/m; > limit_req_zone $limit_premium zone=premium:10m rate=1r/s; > > limit_req zone=free; > limit_req zone=basic; > limit_req zone=premium; > > From the example it is not very clear how the suggested change is > beneficial." The issue with this approach (which is a similar set up to the one that prompted this patch) is the scalability of the solution -  one zone is required per unique rate, and the inability to assign arbitrary rates at runtime. Perhaps a better example than the first one I gave to illustrate the full utility would be with this scenario - involving a user sending a JWT to determine the rate limit that is applied: The limit_req_zone is assigned with $jwt_claim_sub as the key and $jwt_claim_rate as the rate. A user presents the following JWT with every request, his requests now have a rate limit of 100r/s. {     ...     "sub": "user01",     "rate": "100r/s" } Mid-session, that user then acquires a new JWT to get increased speed -  the limit associated with this state/node is now 1000r/s with the same key(sub). Note that these rate values do not need to be preset in the nginx configuration, and it can all be done with a single limit_req_zone. {     ...     "sub": "user01",     "rate": "1000r/s" } Similar could also be done with key_val zone (ie. adding and modifying rate strings at runtime within a key_val zone, and doing a lookup in the key_val zone for that value based upon a user identifier or group as the key). > "Note that this approach implies parsing rate on each request. > This is something we usually try to avoid as long as a static > string is configured in advance, see these commits for an example: > > changeset: 7503:b82162b8496a > user: Ruslan Ermilov > date: Wed Apr 24 16:38:51 2019 +0300 > summary: Added ngx_http_set_complex_value_size_slot(). > > changeset: 7504:c19ca381b2e6 > user: Ruslan Ermilov > date: Wed Apr 24 16:38:54 2019 +0300 > summary: Variables support in limit_rate and limit_rate_after (ticket #293). > > changeset: 7505:16a1adadf437 > user: Ruslan Ermilov > date: Wed Apr 24 16:38:56 2019 +0300 > summary: Variables support in proxy_upload_rate and proxy_download_rate. > > A more self-contained solution can be seen in the image filter > (src/http/modules/ngx_http_image_filter.c), which uses values > parsed during configuration parsing if there are no variables, and > complex values if there are variables (e.g., "imcf->angle" and > "imcf->acv")." Thanks, that makes perfect sense. I'll update accordingly. > Note that this is a style change, and it is wrong. > > The same here. > > Note that this is a style change, since the max_delay is set in > all possible code paths, and there are no reasons to additionally > initialize it. > > Note that this is a style change, since the max_delay is set in > all possible code paths, and there are no reasons to additionally > initialize it. > > See above. Understood, I'll work to fix the style of my changes and I will keep it more consistent with the guidelines in future. > This seems to be separate change, irrelevant to what the patch > does. If at all, it should be submitted as a separate patch, see > http://nginx.org/en/docs/contributing_changes.html. > > Note though that this changes looks wrong to me, as it will break > request accounting on large time intervals with large burst, such > as with rate=1r/m burst=100. > > Note well that it fails to use an empty line before "} else ", > which is a style bug. You are correct, it will break for that scenario. I will rework this and the related portions to more generic overflow detection & handling. I did include it in this patch as there is a danger (which is already present) of rate * ms overflowing. rate is only limited by atoi's limits at present(and it's multiplied by 1000 after that even) . This is more of a worrisome when arbitrary values (including user request values) are in use place of static values from the nginx configuration. I will make those changes into a separate patch as advised. > The "lr->rate", which is saved in the shared memory, seems to be > never used except during processing of the same request. Any > reasons to keep it in the shared memory, reducing the number of > states which can be stored there? A persistent record of the rate that was last assigned to a given node/state is needed exclusively for expiring records (in the expire function). I don't believe this can be avoided, as the rate of the current request is unrelated to the 'to be removed' record's. Thanks again. On 01/01/2023 17:35, Maxim Dounin wrote: > Hello! > > On Fri, Dec 30, 2022 at 10:23:12PM +0000, J Carter wrote: > >> Please find below a patch to enable dynamic rate limiting for >> limit_req module. > Thanks for the patch, and Happy New Year. > > The fundamental problem with dynamic rates in limit_req, which is > a leaky bucket limiter, is that the rate is a property which > applies to multiple requests as accumulated in the bucket (and > this is basically why it is configured in the limit_req_zone, and > not in limit_req), while the dynamically calculated rate is a > property of the last request. This can easily result in > counter-intuitive behaviour. > > For example, consider 5 requests with 1 second interval, assuming > burst 2, and rate being 1r/m or 100r/s depending on some request > properties: > > - 1st request, rate 1r/m, request is accepted > - 2st request, rate 1r/m, request is accepted > - 3rd request, rate 1r/m, request is rejected, since there is no > room in the bucket > - 4th request, rate 100r/s, request is accepted > - 5th request, rate 1r/m, request is accepted (unexpectedly) > > Note that the 5th request is accepted, while it is mostly > equivalent to the 3rd request, and one could expect it to be > rejected. But it happens to to be accepted because the 4th > request "cleared" the bucket. > > Could you please clarify how this problem is addressed in your > patch? > > Note well that the same limits now can be configured separately, > in distinct shared memory zones. This makes it possible to limit > them requests consistently, without any unexpected deviations from > the configured behaviour if requests with different rates > interfere. > >> /* ----------------------------EXAMPLE---------------------------------*/ >> >> geo $traffic_tier { >> default free; >> 127.0.1.0/24 basic; >> 127.0.2.0/24 premium; >> } >> >> map $traffic_tier $rate { >> free 1r/m; >> basic 2r/m; >> premium 1r/s; >> } >> >> limit_req_zone $binary_remote_addr zone=one:10m rate=$rate; >> >> server { >> limit_req zone=one; >> listen 80; >> server_name localhost; >> return 200; >> } >> >> curl --interface 127.0.X.X localhost > Just in case, the very same behaviour can be implemented with > multiple limit_req_zones, with something like this: > > map $traffic_tier $limit_free { > free $binary_remote_addr; > } > > map $traffic_tier $limit_basic { > basic $binary_remote_addr; > } > > map $traffic_tier $limit_premium { > premium $binary_remote_addr; > } > > limit_req_zone $limit_free zone=free:10m rate=1r/m; > limit_req_zone $limit_basic zone=basic:10m rate=2r/m; > limit_req_zone $limit_premium zone=premium:10m rate=1r/s; > > limit_req zone=free; > limit_req zone=basic; > limit_req zone=premium; > > From the example it is not very clear how the suggested change is > beneficial. > > See below for some comments about the code. > > [...] > >> # HG changeset patch >> # User jordanc.carter at outlook.com >> # Date 1672437935 0 >> # Fri Dec 30 22:05:35 2022 +0000 >> # Branch dynamic-rate-limiting >> # Node ID b2bd50efa81e5aeeb9b8f84ee0af34463add07fa >> # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 >> Changed 'rate=' to complex value and added limits to the rate value to prevent integer overflow/underflow >> >> diff -r 07b0bee87f32 -r b2bd50efa81e src/http/modules/ngx_http_limit_req_module.c >> --- a/src/http/modules/ngx_http_limit_req_module.c Wed Dec 21 14:53:27 2022 +0300 >> +++ b/src/http/modules/ngx_http_limit_req_module.c Fri Dec 30 22:05:35 2022 +0000 >> @@ -26,6 +26,7 @@ >> /* integer value, 1 corresponds to 0.001 r/s */ >> ngx_uint_t excess; >> ngx_uint_t count; >> + ngx_uint_t rate; >> u_char data[1]; >> } ngx_http_limit_req_node_t; >> >> @@ -41,7 +42,7 @@ >> ngx_http_limit_req_shctx_t *sh; >> ngx_slab_pool_t *shpool; >> /* integer value, 1 corresponds to 0.001 r/s */ >> - ngx_uint_t rate; >> + ngx_http_complex_value_t rate; >> ngx_http_complex_value_t key; >> ngx_http_limit_req_node_t *node; >> } ngx_http_limit_req_ctx_t; >> @@ -66,9 +67,9 @@ >> >> static void ngx_http_limit_req_delay(ngx_http_request_t *r); >> static ngx_int_t ngx_http_limit_req_lookup(ngx_http_limit_req_limit_t *limit, >> - ngx_uint_t hash, ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account); >> + ngx_uint_t hash, ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account, ngx_uint_t rate); >> static ngx_msec_t ngx_http_limit_req_account(ngx_http_limit_req_limit_t *limits, >> - ngx_uint_t n, ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit); >> + ngx_uint_t n, ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit, ngx_uint_t rate); >> static void ngx_http_limit_req_unlock(ngx_http_limit_req_limit_t *limits, >> ngx_uint_t n); >> static void ngx_http_limit_req_expire(ngx_http_limit_req_ctx_t *ctx, >> @@ -195,10 +196,13 @@ >> ngx_http_limit_req_handler(ngx_http_request_t *r) >> { >> uint32_t hash; >> - ngx_str_t key; >> + ngx_str_t key, rate_s; >> ngx_int_t rc; >> ngx_uint_t n, excess; >> + ngx_uint_t scale; >> + ngx_uint_t rate; >> ngx_msec_t delay; >> + u_char *p; >> ngx_http_limit_req_ctx_t *ctx; >> ngx_http_limit_req_conf_t *lrcf; >> ngx_http_limit_req_limit_t *limit, *limits; >> @@ -243,10 +247,34 @@ >> >> hash = ngx_crc32_short(key.data, key.len); >> >> + if (ngx_http_complex_value(r, &ctx->rate, &rate_s) != NGX_OK) { >> + ngx_http_limit_req_unlock(limits, n); >> + return NGX_HTTP_INTERNAL_SERVER_ERROR; >> + } >> + >> + scale = 1; >> + rate = NGX_ERROR; >> + >> + if (rate_s.len > 8) { >> + >> + rate = (ngx_uint_t) ngx_atoi(rate_s.data + 5, rate_s.len - 8); >> + >> + p = rate_s.data + rate_s.len - 3; >> + if (ngx_strncmp(p, "r/m", 3) == 0) { >> + scale = 60; >> + } else if (ngx_strncmp(p, "r/s", 3) != 0){ >> + rate = NGX_ERROR; >> + } >> + } > Note that this approach implies parsing rate on each request. > This is something we usually try to avoid as long as a static > string is configured in advance, see these commits for an example: > > changeset: 7503:b82162b8496a > user: Ruslan Ermilov > date: Wed Apr 24 16:38:51 2019 +0300 > summary: Added ngx_http_set_complex_value_size_slot(). > > changeset: 7504:c19ca381b2e6 > user: Ruslan Ermilov > date: Wed Apr 24 16:38:54 2019 +0300 > summary: Variables support in limit_rate and limit_rate_after (ticket #293). > > changeset: 7505:16a1adadf437 > user: Ruslan Ermilov > date: Wed Apr 24 16:38:56 2019 +0300 > summary: Variables support in proxy_upload_rate and proxy_download_rate. > > A more self-contained solution can be seen in the image filter > (src/http/modules/ngx_http_image_filter.c), which uses values > parsed during configuration parsing if there are no variables, and > complex values if there are variables (e.g., "imcf->angle" and > "imcf->acv"). > >> + >> + rate = (rate != 0 && rate < NGX_MAX_INT_T_VALUE / 60000000 - 1001) ? >> + rate * 1000 / scale : >> + NGX_MAX_INT_T_VALUE / 60000000 - 1001; >> + >> ngx_shmtx_lock(&ctx->shpool->mutex); >> >> rc = ngx_http_limit_req_lookup(limit, hash, &key, &excess, >> - (n == lrcf->limits.nelts - 1)); >> + (n == lrcf->limits.nelts - 1), rate); >> >> ngx_shmtx_unlock(&ctx->shpool->mutex); >> >> @@ -291,7 +319,7 @@ >> excess = 0; >> } >> >> - delay = ngx_http_limit_req_account(limits, n, &excess, &limit); >> + delay = ngx_http_limit_req_account(limits, n, &excess, &limit, rate); >> >> if (!delay) { >> r->main->limit_req_status = NGX_HTTP_LIMIT_REQ_PASSED; >> @@ -403,7 +431,7 @@ >> >> static ngx_int_t >> ngx_http_limit_req_lookup(ngx_http_limit_req_limit_t *limit, ngx_uint_t hash, >> - ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account) >> + ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account, ngx_uint_t rate) >> { >> size_t size; >> ngx_int_t rc, excess; >> @@ -412,7 +440,6 @@ >> ngx_rbtree_node_t *node, *sentinel; >> ngx_http_limit_req_ctx_t *ctx; >> ngx_http_limit_req_node_t *lr; >> - >> now = ngx_current_msec; >> >> ctx = limit->shm_zone->data; > Note that this is a style change, and it is wrong. > >> @@ -446,12 +473,14 @@ >> >> if (ms < -60000) { >> ms = 1; >> - > The same here. > >> } else if (ms < 0) { >> ms = 0; >> + } else if (ms > 60000) { >> + ms = 60000; >> } > This seems to be separate change, irrelevant to what the patch > does. If at all, it should be submitted as a separate patch, see > http://nginx.org/en/docs/contributing_changes.html. > > Note though that this changes looks wrong to me, as it will break > request accounting on large time intervals with large burst, such > as with rate=1r/m burst=100. > > Note well that it fails to use an empty line before "} else ", > which is a style bug. > >> - excess = lr->excess - ctx->rate * ms / 1000 + 1000; >> + lr->rate = rate; >> + excess = lr->excess - lr->rate * ms / 1000 + 1000; >> >> if (excess < 0) { >> excess = 0; >> @@ -510,6 +539,7 @@ >> >> lr->len = (u_short) key->len; >> lr->excess = 0; >> + lr->rate = rate; > The "lr->rate", which is saved in the shared memory, seems to be > never used except during processing of the same request. Any > reasons to keep it in the shared memory, reducing the number of > states which can be stored there? > >> ngx_memcpy(lr->data, key->data, key->len); >> >> @@ -534,7 +564,7 @@ >> >> static ngx_msec_t >> ngx_http_limit_req_account(ngx_http_limit_req_limit_t *limits, ngx_uint_t n, >> - ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit) >> + ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit, ngx_uint_t rate) >> { >> ngx_int_t excess; >> ngx_msec_t now, delay, max_delay; >> @@ -543,13 +573,13 @@ >> ngx_http_limit_req_node_t *lr; >> >> excess = *ep; >> + max_delay = 0; >> > Note that this is a style change, since the max_delay is set in > all possible code paths, and there are no reasons to additionally > initialize it. > >> if ((ngx_uint_t) excess <= (*limit)->delay) { >> max_delay = 0; >> >> } else { >> - ctx = (*limit)->shm_zone->data; >> - max_delay = (excess - (*limit)->delay) * 1000 / ctx->rate; >> + max_delay = (excess - (*limit)->delay) * 1000 / rate; >> } >> >> while (n--) { >> @@ -570,9 +600,11 @@ >> >> } else if (ms < 0) { >> ms = 0; >> + } else if (ms > 60000) { >> + ms = 60000; >> } > See above. > > [...] > From mdounin at mdounin.ru Mon Jan 2 23:47:15 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 3 Jan 2023 02:47:15 +0300 Subject: [PATCH] Dynamic rate limiting for limit_req module In-Reply-To: References: Message-ID: Hello! On Mon, Jan 02, 2023 at 06:21:03AM +0000, J Carter wrote: > > "The fundamental problem with dynamic rates in limit_req, which is > > a leaky bucket limiter, is that the rate is a property which > > applies to multiple requests as accumulated in the bucket (and > > this is basically why it is configured in the limit_req_zone, and > > not in limit_req), while the dynamically calculated rate is a > > property of the last request. > > > > This can easily result in counter-intuitive behaviour. > > For example, consider 5 requests with 1 second interval, assuming > > burst 2, and rate being 1r/m or 100r/s depending on some request > > properties: > > > > - 1st request, rate 1r/m, request is accepted > > - 2st request, rate 1r/m, request is accepted > > - 3rd request, rate 1r/m, request is rejected, since there is no > > room in the bucket > > - 4th request, rate 100r/s, request is accepted > > - 5th request, rate 1r/m, request is accepted (unexpectedly) > > > > Note that the 5th request is accepted, while it is mostly > > equivalent to the 3rd request, and one could expect it to be > > rejected. But it happens to to be accepted because the 4th > > request "cleared" the bucket." > > This is true - however I do wonder if the behavior (clearing the bucket) > is truly counter intuitive in that scenario. Well, certainly it is. And it is even more so, if you'll assume that 4th and 5th requests are processed essentially at the same time: it will be a race condition with different outcomes depending on the actual order of the requests during limit_req zone updates. For additional fun, this order is not the same as order of the requests in logs. > Should it not be expected that this user/key/state that has been > assigned 100r/s (even if just for one request) would have it's > outstanding excess requests recalculated (and in this case absolved) by > that rate increase? > You have after all assigned it more capacity. > > I'm not sure how to elegantly avoid this if it is an issue, since there > is no 'standard' rate to reference (it could interpolate over > time/requests, but that might be even more confusing). I don't think there is a good solution, and that's one of the reasons why there is no such functionality in limit_req. Instead, it provides a fixed rate, and an ability to provide different burst limits for different requests (which is checked on a per-request basis), as well as multiple different limits to cover cases when multiple rates are needed. > > "Note well that the same limits now can be configured separately, > > in distinct shared memory zones. This makes it possible to limit > > them requests consistently, without any unexpected deviations from > > the configured behaviour if requests with different rates > > interfere. > > > > Just in case, the very same behaviour can be implemented with > > multiple limit_req_zones, with something like this: > > > > map $traffic_tier $limit_free { > > free $binary_remote_addr; > > } > > > > map $traffic_tier $limit_basic { > > basic $binary_remote_addr; > > } > > > > map $traffic_tier $limit_premium { > > premium $binary_remote_addr; > > } > > > > limit_req_zone $limit_free zone=free:10m rate=1r/m; > > limit_req_zone $limit_basic zone=basic:10m rate=2r/m; > > limit_req_zone $limit_premium zone=premium:10m rate=1r/s; > > > > limit_req zone=free; > > limit_req zone=basic; > > limit_req zone=premium; > > > > From the example it is not very clear how the suggested change is > > beneficial." > > The issue with this approach (which is a similar set up to the one that > prompted this patch) is the scalability of the solution -  one zone is > required per unique rate, and the inability to assign arbitrary rates at > runtime. Yet the solution is exactly equivalent to the example provided. Further, given that the use case clearly do no match the intended usage of limit_req, which is designed to as a simple DoS-mitigation and bruteforce-mitigation solution, it looks pretty match trivial to implement. > Perhaps a better example than the first one I gave to illustrate the > full utility would be with this scenario - involving a user sending a > JWT to determine the rate limit that is applied: > > The limit_req_zone is assigned with $jwt_claim_sub as the key and > $jwt_claim_rate as the rate. > > A user presents the following JWT with every request, his requests now > have a rate limit of 100r/s. > { >     ... >     "sub": "user01", >     "rate": "100r/s" > } > Mid-session, that user then acquires a new JWT to get increased speed -  > the limit associated with this state/node is now 1000r/s with the same > key(sub). Note that these rate values do not need to be preset in the > nginx configuration, and it can all be done with a single limit_req_zone. > { >     ... >     "sub": "user01", >     "rate": "1000r/s" > } > Similar could also be done with key_val zone (ie. adding and modifying > rate strings at runtime within a key_val zone, and doing a lookup in the > key_val zone for that value based upon a user identifier or group as the > key). This looks exactly equivalent to the configuration previously discussed. Note well that actually having proper nginx configuration for all the allowed rate values ensure that no invalid and/or arbitrary rates can be supplied by users, providing an additional consistency check. Also note that this is not how limit_req is expected to be used. As already mentioned above, limit_req is a DoS and bruteforce mitigation solution rather than a solution for billing enforcement. [...] > > The "lr->rate", which is saved in the shared memory, seems to be > > never used except during processing of the same request. Any > > reasons to keep it in the shared memory, reducing the number of > > states which can be stored there? > > A persistent record of the rate that was last assigned to a given > node/state is needed exclusively for expiring records (in the expire > function). I don't believe this can be avoided, as the rate of the > current request is unrelated to the 'to be removed' record's. Ah, sorry, missed this. So indeed, it needs to be explicitly stored in the shared zone then (if at all). [...] -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Tue Jan 3 02:48:36 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Tue, 03 Jan 2023 05:48:36 +0300 Subject: [PATCH] Gzip static: ranges support (ticket #2349) Message-ID: # HG changeset patch # User Maxim Dounin # Date 1672713976 -10800 # Tue Jan 03 05:46:16 2023 +0300 # Node ID e0688b4494f02dcf6feebf0c73e02749bd7de381 # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 Gzip static: ranges support (ticket #2349). In contrast to on-the-fly gzipping with gzip filter, static gzipped representation as returned by gzip_static is persistent, and therefore the same binary representation is available for future requests, making it possible to use range requests. Further, if a gzipped representation is re-generated with different compression settings, it is expected to result in different ETag and different size reported in the Content-Range header, making it possible to safely use range requests anyway. As such, ranges are now allowed for files returned by gzip_static. diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c --- a/src/http/modules/ngx_http_gzip_static_module.c +++ b/src/http/modules/ngx_http_gzip_static_module.c @@ -247,6 +247,8 @@ ngx_http_gzip_static_handler(ngx_http_re ngx_str_set(&h->value, "gzip"); r->headers_out.content_encoding = h; + r->allow_ranges = 1; + /* we need to allocate all before the header would be sent */ b = ngx_calloc_buf(r->pool); From jordanc.carter at outlook.com Tue Jan 3 06:25:07 2023 From: jordanc.carter at outlook.com (J Carter) Date: Tue, 3 Jan 2023 06:25:07 +0000 Subject: [PATCH] Dynamic rate limiting for limit_req module In-Reply-To: References: Message-ID: Hello, Thanks again for your reply and thoughts. On 02/01/2023 23:47, Maxim Dounin wrote: > Hello! > > On Mon, Jan 02, 2023 at 06:21:03AM +0000, J Carter wrote: > >>> "The fundamental problem with dynamic rates in limit_req, which is >>> a leaky bucket limiter, is that the rate is a property which >>> applies to multiple requests as accumulated in the bucket (and >>> this is basically why it is configured in the limit_req_zone, and >>> not in limit_req), while the dynamically calculated rate is a >>> property of the last request. >>> >>> This can easily result in counter-intuitive behaviour. >>> For example, consider 5 requests with 1 second interval, assuming >>> burst 2, and rate being 1r/m or 100r/s depending on some request >>> properties: >>> >>> - 1st request, rate 1r/m, request is accepted >>> - 2st request, rate 1r/m, request is accepted >>> - 3rd request, rate 1r/m, request is rejected, since there is no >>> room in the bucket >>> - 4th request, rate 100r/s, request is accepted >>> - 5th request, rate 1r/m, request is accepted (unexpectedly) >>> >>> Note that the 5th request is accepted, while it is mostly >>> equivalent to the 3rd request, and one could expect it to be >>> rejected. But it happens to to be accepted because the 4th >>> request "cleared" the bucket." >> This is true - however I do wonder if the behavior (clearing the bucket) >> is truly counter intuitive in that scenario. > Well, certainly it is. And it is even more so, if you'll assume > that 4th and 5th requests are processed essentially at the same > time: it will be a race condition with different outcomes > depending on the actual order of the requests during limit_req > zone updates. For additional fun, this order is not the same as > order of the requests in logs. You're right, it's a good point. Given a short duration between requests, the behavior during transitions would not be deterministic and could 'flip flop' with regards to the excess value. This behavior could be harsh in the extreme rate jump scenario presented. > >> Should it not be expected that this user/key/state that has been >> assigned 100r/s (even if just for one request) would have it's >> outstanding excess requests recalculated (and in this case absolved) by >> that rate increase? >> You have after all assigned it more capacity. >> >> I'm not sure how to elegantly avoid this if it is an issue, since there >> is no 'standard' rate to reference (it could interpolate over >> time/requests, but that might be even more confusing). > I don't think there is a good solution, and that's one of the > reasons why there is no such functionality in limit_req. Instead, > it provides a fixed rate, and an ability to provide different > burst limits for different requests (which is checked on a > per-request basis), as well as multiple different limits to cover > cases when multiple rates are needed. I have a couple of ideas to solve this issue, that I plan to investigate further: Interpolation - Instead of transitioning 'rate' in a single request straight to a new value, do it smoothly over time/requests. In practical terms this would be achieved by interpolating the state's new rate with the previous rate, and using time between requests for that state (ms) as a scale factor. A new command variable would be added to control the smoothness(the slope angle) of the interpolation. Since interpolation is an additive process, and considers time, this would smooth 'excess''s change and also mask the order of concurrent operations/requests (so no race). Another option - Create a new state per unique 'key and rate' combination. This is effectively the same logical process(and memory required) as with the manual approach from your example (just dynamically determined at runtime in a single zone). This means when a user (a given key) changes rate, a new state is created. The old state remains - to be cleaned up later via expire, and conversely if the user switches back to their original rate before expiry the old state will become active once again. This would mean concatenating key and rate's values for the hash used for the state (as it must be unique). This would increase memory usage briefly during the transition periods due to the additional states and would not consider the other key + rate combo's state's excess value when moving between rates (again much the same as the manual approach). The first option is most suitable to a 'fine grained' key (per user) rate limiting, whereas the second is more suitable for a 'broad based' key (one with lots of different rates). >>> "Note well that the same limits now can be configured separately, >>> in distinct shared memory zones. This makes it possible to limit >>> them requests consistently, without any unexpected deviations from >>> the configured behaviour if requests with different rates >>> interfere. >>> >>> Just in case, the very same behaviour can be implemented with >>> multiple limit_req_zones, with something like this: >>> >>> map $traffic_tier $limit_free { >>> free $binary_remote_addr; >>> } >>> >>> map $traffic_tier $limit_basic { >>> basic $binary_remote_addr; >>> } >>> >>> map $traffic_tier $limit_premium { >>> premium $binary_remote_addr; >>> } >>> >>> limit_req_zone $limit_free zone=free:10m rate=1r/m; >>> limit_req_zone $limit_basic zone=basic:10m rate=2r/m; >>> limit_req_zone $limit_premium zone=premium:10m rate=1r/s; >>> >>> limit_req zone=free; >>> limit_req zone=basic; >>> limit_req zone=premium; >>> >>> From the example it is not very clear how the suggested change is >>> beneficial." >> The issue with this approach (which is a similar set up to the one that >> prompted this patch) is the scalability of the solution -  one zone is >> required per unique rate, and the inability to assign arbitrary rates at >> runtime. > Yet the solution is exactly equivalent to the example provided. > > Further, given that the use case clearly do no match the intended > usage of limit_req, which is designed to as a simple > DoS-mitigation and bruteforce-mitigation solution, it looks pretty > match trivial to implement. It is trivial for a handful of states, however a single map can neatly contain 100s of such rate mappings. It seems preferable to have one or two maps with a single limit_req_zone & limit_req directive as opposed to 100x of each (as well as having to ensure each zone is allocated sufficient memory, as opposed to simply monitoring a single zone). >> Perhaps a better example than the first one I gave to illustrate the >> full utility would be with this scenario - involving a user sending a >> JWT to determine the rate limit that is applied: >> >> The limit_req_zone is assigned with $jwt_claim_sub as the key and >> $jwt_claim_rate as the rate. >> >> A user presents the following JWT with every request, his requests now >> have a rate limit of 100r/s. >> { >>     ... >>     "sub": "user01", >>     "rate": "100r/s" >> } >> Mid-session, that user then acquires a new JWT to get increased speed - >> the limit associated with this state/node is now 1000r/s with the same >> key(sub). Note that these rate values do not need to be preset in the >> nginx configuration, and it can all be done with a single limit_req_zone. >> { >>     ... >>     "sub": "user01", >>     "rate": "1000r/s" >> } >> Similar could also be done with key_val zone (ie. adding and modifying >> rate strings at runtime within a key_val zone, and doing a lookup in the >> key_val zone for that value based upon a user identifier or group as the >> key). > This looks exactly equivalent to the configuration previously > discussed. This short example is, but see the previous issue. In theory you could have a thousands of individual rates(tailored per user) with this method with zero additional nginx configuration. > > Note well that actually having proper nginx configuration for all > the allowed rate values ensure that no invalid and/or arbitrary > rates can be supplied by users, providing an additional > consistency check. I avoided mentioning 'untrusted request headers' (cookies) and went with JWT claims for this reason. Since you can cryptographically verify a JWTs authenticity, you know if it's been tampered with - you can determine if it still contains the original rate assigned by the administrator controlled application that generated the token). It is the duty of the administrator to set an appropriate rate value in that token. It's possible to handle JWT in pure NJS (https://github.com/lombax85/nginx-jwt) -  I'll put something together to illustrate this better with a full example. > > Also note that this is not how limit_req is expected to be used. > As already mentioned above, limit_req is a DoS and bruteforce > mitigation solution rather than a solution for billing > enforcement. I do agree - it an extension to it's intended purpose, however dynamic request rates is consistent with other capabilities in nginx. https://nginx.org/en/docs/http/ngx_http_core_module.html#limit_rate Perhaps the proposed changes is/will be to far removed from the original purpose of the module - in which case I will make the required alterations and resubmit it as a separate module for consideration. From mdounin at mdounin.ru Tue Jan 3 22:11:27 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Wed, 04 Jan 2023 01:11:27 +0300 Subject: [PATCH 2 of 2] Fixed article in "if_modified_since before" In-Reply-To: <52ea1f45b324b404941a.1672783886@vm-bsd.mdounin.ru> References: <52ea1f45b324b404941a.1672783886@vm-bsd.mdounin.ru> Message-ID: <547303412ab9bf1b2282.1672783887@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1672783715 -10800 # Wed Jan 04 01:08:35 2023 +0300 # Node ID 547303412ab9bf1b22820b459143bd48d051f92f # Parent 52ea1f45b324b404941a9b31b2861612cf090fcd Fixed article in "if_modified_since before". diff --git a/xml/en/docs/http/ngx_http_core_module.xml b/xml/en/docs/http/ngx_http_core_module.xml --- a/xml/en/docs/http/ngx_http_core_module.xml +++ b/xml/en/docs/http/ngx_http_core_module.xml @@ -802,7 +802,7 @@ exact match; before -modification time of a response is +modification time of the response is less than or equal to the time in the
If-Modified-Since
request header field. From mdounin at mdounin.ru Tue Jan 3 22:11:26 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Wed, 04 Jan 2023 01:11:26 +0300 Subject: [PATCH 1 of 2] Clarified "if_modified_since off" behaviour (ticket #2176) Message-ID: <52ea1f45b324b404941a.1672783886@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1672783626 -10800 # Wed Jan 04 01:07:06 2023 +0300 # Node ID 52ea1f45b324b404941a9b31b2861612cf090fcd # Parent 8033ffaedeb9f029897d464d85454b9cffa35fd9 Clarified "if_modified_since off" behaviour (ticket #2176). diff --git a/xml/en/docs/http/ngx_http_core_module.xml b/xml/en/docs/http/ngx_http_core_module.xml --- a/xml/en/docs/http/ngx_http_core_module.xml +++ b/xml/en/docs/http/ngx_http_core_module.xml @@ -10,7 +10,7 @@ + rev="103">
@@ -792,8 +792,7 @@ request header field: off -the -
If-Modified-Since
request header field is ignored (0.7.34); +the response is always considered modified (0.7.34); exact diff --git a/xml/ru/docs/http/ngx_http_core_module.xml b/xml/ru/docs/http/ngx_http_core_module.xml --- a/xml/ru/docs/http/ngx_http_core_module.xml +++ b/xml/ru/docs/http/ngx_http_core_module.xml @@ -10,7 +10,7 @@ + rev="103">
@@ -787,8 +787,7 @@ error_page 404 =301 http://example.com/n off -не проверять поле -
If-Modified-Since
заголовка запроса (0.7.34); +ответ всегда считается изменившимся (0.7.34); exact From yar at nginx.com Wed Jan 4 11:32:13 2023 From: yar at nginx.com (Yaroslav Zhuravlev) Date: Wed, 4 Jan 2023 11:32:13 +0000 Subject: [PATCH] Updated advisory links In-Reply-To: <165e9f39cfc8815cfc18.1672600715@vm-bsd.mdounin.ru> References: <165e9f39cfc8815cfc18.1672600715@vm-bsd.mdounin.ru> Message-ID: <88F5BB20-41E7-4A4B-82A4-086FC93068DA@nginx.com> > On 1 Jan 2023, at 19:18, Maxim Dounin wrote: > > # HG changeset patch > # User Maxim Dounin > # Date 1672600531 -10800 > # Sun Jan 01 22:15:31 2023 +0300 > # Node ID 165e9f39cfc8815cfc18b7cf6cbf712059585608 > # Parent 8033ffaedeb9f029897d464d85454b9cffa35fd9 > Updated advisory links. > > Mailman 3 was unusable, and therefore list software was changed back to > Mailman 2, with all the archives imported into pipermail. Advisory > links updated accordingly. > > diff --git a/xml/en/security_advisories.xml b/xml/en/security_advisories.xml > --- a/xml/en/security_advisories.xml > +++ b/xml/en/security_advisories.xml > @@ -26,7 +26,7 @@ Patches are signed using one of the > > severity="medium" > - advisory="https://mailman.nginx.org/archives/list/nginx-announce at nginx.org/message/RBRRON6PYBJJM2XIAPQBFBVLR4Q6IHRA/" > + advisory="http://mailman.nginx.org/pipermail/nginx-announce/2022/RBRRON6PYBJJM2XIAPQBFBVLR4Q6IHRA.html" > cve="2022-41741" > good="1.23.2+, 1.22.1+" > vulnerable="1.1.3-1.23.1, 1.0.7-1.0.15"> > @@ -35,7 +35,7 @@ Patches are signed using one of the > > severity="medium" > - advisory="https://mailman.nginx.org/archives/list/nginx-announce at nginx.org/message/RBRRON6PYBJJM2XIAPQBFBVLR4Q6IHRA/" > + advisory="http://mailman.nginx.org/pipermail/nginx-announce/2022/RBRRON6PYBJJM2XIAPQBFBVLR4Q6IHRA.html" > cve="2022-41742" > good="1.23.2+, 1.22.1+" > vulnerable="1.1.3-1.23.1, 1.0.7-1.0.15"> Thank you, added to nginx.org: http://hg.nginx.org/nginx.org/rev/ef7f2666cc09 > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel From yar at nginx.com Wed Jan 4 11:25:35 2023 From: yar at nginx.com (Yaroslav Zhuravlev) Date: Wed, 4 Jan 2023 11:25:35 +0000 Subject: [PATCH 1 of 2] Clarified "if_modified_since off" behaviour (ticket #2176) In-Reply-To: <52ea1f45b324b404941a.1672783886@vm-bsd.mdounin.ru> References: <52ea1f45b324b404941a.1672783886@vm-bsd.mdounin.ru> Message-ID: > On 3 Jan 2023, at 22:11, Maxim Dounin wrote: > > # HG changeset patch > # User Maxim Dounin > # Date 1672783626 -10800 > # Wed Jan 04 01:07:06 2023 +0300 > # Node ID 52ea1f45b324b404941a9b31b2861612cf090fcd > # Parent 8033ffaedeb9f029897d464d85454b9cffa35fd9 > Clarified "if_modified_since off" behaviour (ticket #2176). > > diff --git a/xml/en/docs/http/ngx_http_core_module.xml b/xml/en/docs/http/ngx_http_core_module.xml > --- a/xml/en/docs/http/ngx_http_core_module.xml > +++ b/xml/en/docs/http/ngx_http_core_module.xml > @@ -10,7 +10,7 @@ > link="/en/docs/http/ngx_http_core_module.html" > lang="en" > - rev="102"> > + rev="103"> > >
> > @@ -792,8 +792,7 @@ request header field: > > off > > -the > -
If-Modified-Since
request header field is ignored (0.7.34); > +the response is always considered modified (0.7.34); > > > exact > diff --git a/xml/ru/docs/http/ngx_http_core_module.xml b/xml/ru/docs/http/ngx_http_core_module.xml > --- a/xml/ru/docs/http/ngx_http_core_module.xml > +++ b/xml/ru/docs/http/ngx_http_core_module.xml > @@ -10,7 +10,7 @@ > link="/ru/docs/http/ngx_http_core_module.html" > lang="ru" > - rev="102"> > + rev="103"> > >
> > @@ -787,8 +787,7 @@ error_page 404 =301 http://example.com/n > > off > > -не проверять поле > -
If-Modified-Since
заголовка запроса (0.7.34); > +ответ всегда считается изменившимся (0.7.34); > > > exact Thank you, added to nginx.org: http://hg.nginx.org/nginx.org/rev/52ea1f45b324 > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel From yar at nginx.com Wed Jan 4 11:24:47 2023 From: yar at nginx.com (Yaroslav Zhuravlev) Date: Wed, 4 Jan 2023 11:24:47 +0000 Subject: [PATCH 2 of 2] Fixed article in "if_modified_since before" In-Reply-To: <547303412ab9bf1b2282.1672783887@vm-bsd.mdounin.ru> References: <52ea1f45b324b404941a.1672783886@vm-bsd.mdounin.ru> <547303412ab9bf1b2282.1672783887@vm-bsd.mdounin.ru> Message-ID: <80FEAA26-E03E-4E2E-8B61-7B32AEDA1CCD@nginx.com> > On 3 Jan 2023, at 22:11, Maxim Dounin wrote: > > # HG changeset patch > # User Maxim Dounin > # Date 1672783715 -10800 > # Wed Jan 04 01:08:35 2023 +0300 > # Node ID 547303412ab9bf1b22820b459143bd48d051f92f > # Parent 52ea1f45b324b404941a9b31b2861612cf090fcd > Fixed article in "if_modified_since before". > > diff --git a/xml/en/docs/http/ngx_http_core_module.xml b/xml/en/docs/http/ngx_http_core_module.xml > --- a/xml/en/docs/http/ngx_http_core_module.xml > +++ b/xml/en/docs/http/ngx_http_core_module.xml > @@ -802,7 +802,7 @@ exact match; > > before > > -modification time of a response is > +modification time of the response is > less than or equal to the time in the
If-Modified-Since
> request header field. > Thank you, added to nginx.org: http://hg.nginx.org/nginx.org/rev/547303412ab9 > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel From xeioex at nginx.com Thu Jan 5 04:42:38 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Jan 2023 04:42:38 +0000 Subject: [njs] WebCrypto: fixed importKey() for AES-* keys. Message-ID: details: https://hg.nginx.org/njs/rev/02aa50753dc1 branches: changeset: 2017:02aa50753dc1 user: Dmitry Volyntsev date: Thu Dec 29 20:39:29 2022 -0800 description: WebCrypto: fixed importKey() for AES-* keys. Previously, key of of any length were accepted, whereas according to the spec only 128, 192 and 256 bits are allowed. diffstat: external/njs_webcrypto_module.c | 16 +++++++++++++++- test/webcrypto/aes.t.js | 3 +++ 2 files changed, 18 insertions(+), 1 deletions(-) diffs (55 lines): diff -r 5fc0aa4a4e72 -r 02aa50753dc1 external/njs_webcrypto_module.c --- a/external/njs_webcrypto_module.c Thu Dec 15 13:04:46 2022 +0100 +++ b/external/njs_webcrypto_module.c Thu Dec 29 20:39:29 2022 -0800 @@ -1840,11 +1840,25 @@ njs_ext_import_key(njs_vm_t *vm, njs_val goto fail; } - /* Fall through. */ + key->raw = key_data; + break; case NJS_ALGORITHM_AES_GCM: case NJS_ALGORITHM_AES_CTR: case NJS_ALGORITHM_AES_CBC: + switch (key_data.length) { + case 16: + case 24: + case 32: + break; + + default: + njs_type_error(vm, "Invalid key length"); + goto fail; + } + + /* Fall through. */ + case NJS_ALGORITHM_PBKDF2: case NJS_ALGORITHM_HKDF: key->raw = key_data; diff -r 5fc0aa4a4e72 -r 02aa50753dc1 test/webcrypto/aes.t.js --- a/test/webcrypto/aes.t.js Thu Dec 15 13:04:46 2022 +0100 +++ b/test/webcrypto/aes.t.js Thu Dec 29 20:39:29 2022 -0800 @@ -65,6 +65,7 @@ let aes_tsuite = { { name: "AES-GCM", data: "aabbcc", tagLength: 96 }, { name: "AES-GCM", data: "aabbcc", tagLength: 112 }, { name: "AES-GCM", data: "aabbcc", tagLength: 113, exception: "TypeError: AES-GCM Invalid tagLength" }, + { name: "AES-GCM", data: "aabbcc", key: "aabbcc", exception: "TypeError: Invalid key length" }, { name: "AES-GCM", data: "aabbccdd".repeat(4096) }, { name: "AES-CTR", data: "aa" }, @@ -85,11 +86,13 @@ let aes_tsuite = { { name: "AES-CTR", data: "aabbccdd".repeat(4096), length: 24 }, { name: "AES-CTR", data: "aabbccdd", length: 129, exception: "TypeError: AES-CTR algorithm.length must be between 1 and 128" }, + { name: "AES-CTR", data: "aabbcc", key: "aabbcc", exception: "TypeError: Invalid key length" }, { name: "AES-CBC", data: "aa" }, { name: "AES-CBC", data: "aabbccdd".repeat(4) }, { name: "AES-CBC", data: "aabbccdd".repeat(4096) }, { name: "AES-CBC", data: "aabbccdd".repeat(5), iv: "ffffffffffffffffffffffffffffffff" }, + { name: "AES-CBC", data: "aabbcc", key: "aabbcc", exception: "TypeError: Invalid key length" }, ]}; run([aes_tsuite]) From xeioex at nginx.com Thu Jan 5 04:42:40 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Jan 2023 04:42:40 +0000 Subject: [njs] WebCrypto: added missed support for AES-* keys of size 192. Message-ID: details: https://hg.nginx.org/njs/rev/864bf6445ccb branches: changeset: 2018:864bf6445ccb user: Dmitry Volyntsev date: Thu Dec 29 20:46:21 2022 -0800 description: WebCrypto: added missed support for AES-* keys of size 192. diffstat: external/njs_webcrypto_module.c | 12 ++++++++++++ test/webcrypto/aes.t.js | 3 +++ 2 files changed, 15 insertions(+), 0 deletions(-) diffs (63 lines): diff -r 02aa50753dc1 -r 864bf6445ccb external/njs_webcrypto_module.c --- a/external/njs_webcrypto_module.c Thu Dec 29 20:39:29 2022 -0800 +++ b/external/njs_webcrypto_module.c Thu Dec 29 20:46:21 2022 -0800 @@ -663,6 +663,10 @@ njs_cipher_aes_gcm(njs_vm_t *vm, njs_str cipher = EVP_aes_128_gcm(); break; + case 24: + cipher = EVP_aes_192_gcm(); + break; + case 32: cipher = EVP_aes_256_gcm(); break; @@ -961,6 +965,10 @@ njs_cipher_aes_ctr(njs_vm_t *vm, njs_str cipher = EVP_aes_128_ctr(); break; + case 24: + cipher = EVP_aes_192_ctr(); + break; + case 32: cipher = EVP_aes_256_ctr(); break; @@ -1162,6 +1170,10 @@ njs_cipher_aes_cbc(njs_vm_t *vm, njs_str cipher = EVP_aes_128_cbc(); break; + case 24: + cipher = EVP_aes_192_cbc(); + break; + case 32: cipher = EVP_aes_256_cbc(); break; diff -r 02aa50753dc1 -r 864bf6445ccb test/webcrypto/aes.t.js --- a/test/webcrypto/aes.t.js Thu Dec 29 20:39:29 2022 -0800 +++ b/test/webcrypto/aes.t.js Thu Dec 29 20:46:21 2022 -0800 @@ -66,6 +66,7 @@ let aes_tsuite = { { name: "AES-GCM", data: "aabbcc", tagLength: 112 }, { name: "AES-GCM", data: "aabbcc", tagLength: 113, exception: "TypeError: AES-GCM Invalid tagLength" }, { name: "AES-GCM", data: "aabbcc", key: "aabbcc", exception: "TypeError: Invalid key length" }, + { name: "AES-GCM", data: "aabbcc", key: "001122330011223300112233001122330011223300112233" }, { name: "AES-GCM", data: "aabbccdd".repeat(4096) }, { name: "AES-CTR", data: "aa" }, @@ -84,6 +85,7 @@ let aes_tsuite = { { name: "AES-CTR", data: "aabbccdd".repeat(4096), counter: "ffffffffffffffffffffffffffffffff", length: 11 }, { name: "AES-CTR", data: "aabbccdd".repeat(4096), length: 20 }, { name: "AES-CTR", data: "aabbccdd".repeat(4096), length: 24 }, + { name: "AES-CTR", data: "aabbcc", key: "001122330011223300112233001122330011223300112233" }, { name: "AES-CTR", data: "aabbccdd", length: 129, exception: "TypeError: AES-CTR algorithm.length must be between 1 and 128" }, { name: "AES-CTR", data: "aabbcc", key: "aabbcc", exception: "TypeError: Invalid key length" }, @@ -92,6 +94,7 @@ let aes_tsuite = { { name: "AES-CBC", data: "aabbccdd".repeat(4) }, { name: "AES-CBC", data: "aabbccdd".repeat(4096) }, { name: "AES-CBC", data: "aabbccdd".repeat(5), iv: "ffffffffffffffffffffffffffffffff" }, + { name: "AES-CBC", data: "aabbcc", key: "001122330011223300112233001122330011223300112233" }, { name: "AES-CBC", data: "aabbcc", key: "aabbcc", exception: "TypeError: Invalid key length" }, ]}; From xeioex at nginx.com Thu Jan 5 04:42:42 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Jan 2023 04:42:42 +0000 Subject: [njs] WebCrypto: improved exception wording for invalid key formats. Message-ID: details: https://hg.nginx.org/njs/rev/3e7e2eb6b9aa branches: changeset: 2019:3e7e2eb6b9aa user: Dmitry Volyntsev date: Fri Dec 30 18:22:02 2022 -0800 description: WebCrypto: improved exception wording for invalid key formats. diffstat: external/njs_webcrypto_module.c | 60 +++++++++++++++++++++++----------------- 1 files changed, 35 insertions(+), 25 deletions(-) diffs (107 lines): diff -r 864bf6445ccb -r 3e7e2eb6b9aa external/njs_webcrypto_module.c --- a/external/njs_webcrypto_module.c Thu Dec 29 20:46:21 2022 -0800 +++ b/external/njs_webcrypto_module.c Fri Dec 30 18:22:02 2022 -0800 @@ -122,6 +122,7 @@ static njs_int_t njs_ext_get_random_valu static void njs_webcrypto_cleanup_pkey(void *data); static njs_webcrypto_key_format_t njs_key_format(njs_vm_t *vm, njs_value_t *value); +static njs_str_t *njs_format_string(njs_webcrypto_key_format_t fmt); static njs_int_t njs_key_usage(njs_vm_t *vm, njs_value_t *value, unsigned *mask); static njs_webcrypto_algorithm_t *njs_key_algorithm(njs_vm_t *vm, @@ -278,6 +279,15 @@ static njs_webcrypto_entry_t njs_webcryp }; +static njs_webcrypto_entry_t njs_webcrypto_format[] = { + { njs_str("raw"), NJS_KEY_FORMAT_RAW }, + { njs_str("pkcs8"), NJS_KEY_FORMAT_PKCS8 }, + { njs_str("spki"), NJS_KEY_FORMAT_SPKI }, + { njs_str("jwk"), NJS_KEY_FORMAT_JWK }, + { njs_null_str, NJS_KEY_FORMAT_UNKNOWN } +}; + + static njs_webcrypto_entry_t njs_webcrypto_usage[] = { { njs_str("decrypt"), NJS_KEY_USAGE_DECRYPT }, { njs_str("deriveBits"), NJS_KEY_USAGE_DERIVE_BITS }, @@ -1693,7 +1703,8 @@ njs_ext_import_key(njs_vm_t *vm, njs_val } if (njs_slow_path(!(fmt & alg->fmt))) { - njs_type_error(vm, "unsupported key fmt for \"%V\" key", + njs_type_error(vm, "unsupported key fmt \"%V\" for \"%V\" key", + njs_format_string(fmt), njs_algorithm_string(alg)); goto fail; } @@ -2511,46 +2522,45 @@ njs_webcrypto_cleanup_pkey(void *data) static njs_webcrypto_key_format_t njs_key_format(njs_vm_t *vm, njs_value_t *value) { - njs_int_t ret; - njs_str_t format; - njs_uint_t fmt; - njs_value_t string; - - static const struct { - njs_str_t name; - njs_uint_t value; - } formats[] = { - { njs_str("raw"), NJS_KEY_FORMAT_RAW }, - { njs_str("pkcs8"), NJS_KEY_FORMAT_PKCS8 }, - { njs_str("spki"), NJS_KEY_FORMAT_SPKI }, - { njs_str("jwk"), NJS_KEY_FORMAT_JWK }, - }; + njs_int_t ret; + njs_str_t format; + njs_value_t string; + njs_webcrypto_entry_t *e; ret = njs_value_to_string(vm, &string, value); if (njs_slow_path(ret != NJS_OK)) { - goto fail; + return NJS_KEY_FORMAT_UNKNOWN; } njs_string_get(&string, &format); - fmt = 0; - - while (fmt < sizeof(formats) / sizeof(formats[0])) { - if (njs_strstr_eq(&format, &formats[fmt].name)) { - return formats[fmt].value; + for (e = &njs_webcrypto_format[0]; e->name.length != 0; e++) { + if (njs_strstr_eq(&format, &e->name)) { + return e->value; } - - fmt++; } -fail: - njs_type_error(vm, "unknown key format: \"%V\"", &format); return NJS_KEY_FORMAT_UNKNOWN; } +static njs_str_t * +njs_format_string(njs_webcrypto_key_format_t fmt) +{ + njs_webcrypto_entry_t *e; + + for (e = &njs_webcrypto_format[0]; e->name.length != 0; e++) { + if (fmt == e->value) { + break; + } + } + + return &e->name; +} + + static njs_int_t njs_key_usage_array_handler(njs_vm_t *vm, njs_iterator_args_t *args, njs_value_t *value, int64_t index) From xeioex at nginx.com Thu Jan 5 04:42:44 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Jan 2023 04:42:44 +0000 Subject: [njs] WebCrypto: extended support for asymmetric keys. Message-ID: details: https://hg.nginx.org/njs/rev/0681bf662222 branches: changeset: 2020:0681bf662222 user: Dmitry Volyntsev date: Wed Jan 04 17:49:22 2023 -0800 description: WebCrypto: extended support for asymmetric keys. The following functionality for RSA and EC keys were added: importKey() supporting 'jwk' format, also 'raw' format for EC public keys. exportKey() supporting 'pksc8', 'spki', 'jwk' format, also 'raw' format for EC public keys. generateKey(). diffstat: external/njs_openssl.h | 265 ++++++ external/njs_webcrypto_module.c | 1606 +++++++++++++++++++++++++++++++++++--- test/harness/compareObjects.js | 17 + test/harness/runTsuite.js | 4 +- test/harness/webCryptoUtils.js | 8 + test/ts/test.ts | 12 + test/webcrypto/ec.jwk | 1 + test/webcrypto/ec.pub.jwk | 1 + test/webcrypto/export.t.js | 298 +++++++ test/webcrypto/rsa.dec.jwk | 1 + test/webcrypto/rsa.enc.pub.jwk | 1 + test/webcrypto/rsa.jwk | 1 + test/webcrypto/rsa.pub.jwk | 1 + test/webcrypto/rsa.t.js | 122 ++- test/webcrypto/sign.t.js | 266 ++++++- ts/njs_webcrypto.d.ts | 59 +- 16 files changed, 2471 insertions(+), 192 deletions(-) diffs (truncated from 3178 to 1000 lines): diff -r 3e7e2eb6b9aa -r 0681bf662222 external/njs_openssl.h --- a/external/njs_openssl.h Fri Dec 30 18:22:02 2022 -0800 +++ b/external/njs_openssl.h Wed Jan 04 17:49:22 2023 -0800 @@ -56,4 +56,269 @@ #endif +njs_inline int +njs_bn_bn2binpad(const BIGNUM *bn, unsigned char *to, int tolen) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + return BN_bn2binpad(bn, to, tolen); +#else + return BN_bn2bin(bn, &to[tolen - BN_num_bytes(bn)]); +#endif +} + + +njs_inline int +njs_pkey_up_ref(EVP_PKEY *pkey) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + return EVP_PKEY_up_ref(pkey); +#else + CRYPTO_add(&pkey->references, 1, CRYPTO_LOCK_EVP_PKEY); + return 1; +#endif +} + + +njs_inline const RSA * +njs_pkey_get_rsa_key(EVP_PKEY *pkey) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + return EVP_PKEY_get0_RSA(pkey); +#else + return EVP_PKEY_get0(pkey); +#endif +} + + +njs_inline void +njs_rsa_get0_key(const RSA *rsa, const BIGNUM **n, const BIGNUM **e, + const BIGNUM **d) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + RSA_get0_key(rsa, n, e, d); +#else + if (n != NULL) { + *n = rsa->n; + } + + if (e != NULL) { + *e = rsa->e; + } + + if (d != NULL) { + *d = rsa->d; + } +#endif +} + + +njs_inline void +njs_rsa_get0_factors(const RSA *rsa, const BIGNUM **p, const BIGNUM **q) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + RSA_get0_factors(rsa, p, q); +#else + if (p != NULL) { + *p = rsa->p; + } + + if (q != NULL) { + *q = rsa->q; + } +#endif +} + + + +njs_inline void +njs_rsa_get0_ctr_params(const RSA *rsa, const BIGNUM **dp, const BIGNUM **dq, + const BIGNUM **qi) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + RSA_get0_crt_params(rsa, dp, dq, qi); +#else + if (dp != NULL) { + *dp = rsa->dmp1; + } + + if (dq != NULL) { + *dq = rsa->dmq1; + } + + if (qi != NULL) { + *qi = rsa->iqmp; + } +#endif +} + + +njs_inline int +njs_rsa_set0_key(RSA *rsa, BIGNUM *n, BIGNUM *e, BIGNUM *d) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + return RSA_set0_key(rsa, n, e, d); +#else + if ((rsa->n == NULL && n == NULL) || (rsa->e == NULL && e == NULL)) { + return 0; + } + + if (n != NULL) { + BN_free(rsa->n); + rsa->n = n; + } + + if (e != NULL) { + BN_free(rsa->e); + rsa->e = e; + } + + if (d != NULL) { + BN_clear_free(rsa->d); + rsa->d = d; + BN_set_flags(rsa->d, BN_FLG_CONSTTIME); + } + + return 1; +#endif +} + + +njs_inline int +njs_rsa_set0_factors(RSA *rsa, BIGNUM *p, BIGNUM *q) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + return RSA_set0_factors(rsa, p, q); +#else + if ((rsa->p == NULL && p == NULL) || (rsa->q == NULL && q == NULL)) { + return 0; + } + + if (p != NULL) { + BN_clear_free(rsa->p); + rsa->p = p; + BN_set_flags(rsa->p, BN_FLG_CONSTTIME); + } + + if (q != NULL) { + BN_clear_free(rsa->q); + rsa->q = q; + BN_set_flags(rsa->q, BN_FLG_CONSTTIME); + } + + return 1; +#endif +} + + +njs_inline int +njs_rsa_set0_ctr_params(RSA *rsa, BIGNUM *dp, BIGNUM *dq, BIGNUM *qi) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + return RSA_set0_crt_params(rsa, dp, dq, qi); +#else + if ((rsa->dmp1 == NULL && dp == NULL) + || (rsa->dmq1 == NULL && dq == NULL) + || (rsa->iqmp == NULL && qi == NULL)) + { + return 0; + } + + if (dp != NULL) { + BN_clear_free(rsa->dmp1); + rsa->dmp1 = dp; + BN_set_flags(rsa->dmp1, BN_FLG_CONSTTIME); + } + + if (dq != NULL) { + BN_clear_free(rsa->dmq1); + rsa->dmq1 = dq; + BN_set_flags(rsa->dmq1, BN_FLG_CONSTTIME); + } + + if (qi != NULL) { + BN_clear_free(rsa->iqmp); + rsa->iqmp = qi; + BN_set_flags(rsa->iqmp, BN_FLG_CONSTTIME); + } + + return 1; +#endif +} + + +njs_inline const EC_KEY * +njs_pkey_get_ec_key(EVP_PKEY *pkey) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + return EVP_PKEY_get0_EC_KEY(pkey); +#else + if (pkey->type != EVP_PKEY_EC) { + return NULL; + } + + return pkey->pkey.ec; +#endif +} + + +njs_inline int +njs_ec_group_order_bits(const EC_GROUP *group) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + return EC_GROUP_order_bits(group); +#else + int bits; + BIGNUM *order; + + order = BN_new(); + if (order == NULL) { + return 0; + } + + if (EC_GROUP_get_order(group, order, NULL) == 0) { + return 0; + } + + bits = BN_num_bits(order); + + BN_free(order); + + return bits; +#endif +} + + +njs_inline int +njs_ec_point_get_affine_coordinates(const EC_GROUP *group, const EC_POINT *p, + BIGNUM *x, BIGNUM *y) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100001L) + return EC_POINT_get_affine_coordinates(group, p, x, y, NULL); +#else + return EC_POINT_get_affine_coordinates_GFp(group, p, x, y, NULL); +#endif +} + + +njs_inline int +njs_ecdsa_sig_set0(ECDSA_SIG *sig, BIGNUM *r, BIGNUM *s) +{ +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + return ECDSA_SIG_set0(sig, r, s); +#else + if (r == NULL || s == NULL) { + return 0; + } + + BN_clear_free(sig->r); + BN_clear_free(sig->s); + + sig->r = r; + sig->s = s; + + return 1; +#endif +} + + #endif /* _NJS_EXTERNAL_OPENSSL_H_INCLUDED_ */ diff -r 3e7e2eb6b9aa -r 0681bf662222 external/njs_webcrypto_module.c --- a/external/njs_webcrypto_module.c Fri Dec 30 18:22:02 2022 -0800 +++ b/external/njs_webcrypto_module.c Wed Jan 04 17:49:22 2023 -0800 @@ -32,21 +32,22 @@ typedef enum { typedef enum { + NJS_ALGORITHM_RSASSA_PKCS1_v1_5 = 0, + NJS_ALGORITHM_RSA_PSS, NJS_ALGORITHM_RSA_OAEP, + NJS_ALGORITHM_HMAC, NJS_ALGORITHM_AES_GCM, NJS_ALGORITHM_AES_CTR, NJS_ALGORITHM_AES_CBC, - NJS_ALGORITHM_RSASSA_PKCS1_v1_5, - NJS_ALGORITHM_RSA_PSS, NJS_ALGORITHM_ECDSA, NJS_ALGORITHM_ECDH, NJS_ALGORITHM_PBKDF2, NJS_ALGORITHM_HKDF, - NJS_ALGORITHM_HMAC, } njs_webcrypto_alg_t; typedef enum { + NJS_HASH_UNSET = 0, NJS_HASH_SHA1, NJS_HASH_SHA256, NJS_HASH_SHA384, @@ -54,13 +55,6 @@ typedef enum { } njs_webcrypto_hash_t; -typedef enum { - NJS_CURVE_P256, - NJS_CURVE_P384, - NJS_CURVE_P521, -} njs_webcrypto_curve_t; - - typedef struct { njs_str_t name; uintptr_t value; @@ -76,12 +70,15 @@ typedef struct { typedef struct { njs_webcrypto_algorithm_t *alg; - unsigned usage; njs_webcrypto_hash_t hash; - njs_webcrypto_curve_t curve; + int curve; EVP_PKEY *pkey; njs_str_t raw; + + unsigned usage; + njs_bool_t extractable; + njs_bool_t privat; } njs_webcrypto_key_t; @@ -119,12 +116,14 @@ static njs_int_t njs_ext_wrap_key(njs_vm static njs_int_t njs_ext_get_random_values(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused); -static void njs_webcrypto_cleanup_pkey(void *data); +static njs_webcrypto_key_t *njs_webcrypto_key_alloc(njs_vm_t *vm, + njs_webcrypto_algorithm_t *alg, unsigned usage, njs_bool_t extractable); static njs_webcrypto_key_format_t njs_key_format(njs_vm_t *vm, njs_value_t *value); static njs_str_t *njs_format_string(njs_webcrypto_key_format_t fmt); static njs_int_t njs_key_usage(njs_vm_t *vm, njs_value_t *value, unsigned *mask); +static njs_int_t njs_key_ops(njs_vm_t *vm, njs_value_t *retval, unsigned mask); static njs_webcrypto_algorithm_t *njs_key_algorithm(njs_vm_t *vm, njs_value_t *value); static njs_str_t *njs_algorithm_string(njs_webcrypto_algorithm_t *algorithm); @@ -132,10 +131,12 @@ static njs_int_t njs_algorithm_hash(njs_ njs_webcrypto_hash_t *hash); static const EVP_MD *njs_algorithm_hash_digest(njs_webcrypto_hash_t hash); static njs_int_t njs_algorithm_curve(njs_vm_t *vm, njs_value_t *value, - njs_webcrypto_curve_t *curve); + int *curve); static njs_int_t njs_webcrypto_result(njs_vm_t *vm, njs_value_t *result, njs_int_t rc); +static njs_int_t njs_webcrypto_array_buffer(njs_vm_t *vm, njs_value_t *retval, + u_char *start, size_t length); static void njs_webcrypto_error(njs_vm_t *vm, const char *fmt, ...); static njs_int_t njs_webcrypto_init(njs_vm_t *vm); @@ -154,7 +155,8 @@ static njs_webcrypto_entry_t njs_webcryp NJS_KEY_USAGE_UNWRAP_KEY | NJS_KEY_USAGE_GENERATE_KEY, NJS_KEY_FORMAT_PKCS8 | - NJS_KEY_FORMAT_SPKI) + NJS_KEY_FORMAT_SPKI | + NJS_KEY_FORMAT_JWK) }, { @@ -197,7 +199,8 @@ static njs_webcrypto_entry_t njs_webcryp NJS_KEY_USAGE_VERIFY | NJS_KEY_USAGE_GENERATE_KEY, NJS_KEY_FORMAT_PKCS8 | - NJS_KEY_FORMAT_SPKI) + NJS_KEY_FORMAT_SPKI | + NJS_KEY_FORMAT_JWK) }, { @@ -207,7 +210,8 @@ static njs_webcrypto_entry_t njs_webcryp NJS_KEY_USAGE_VERIFY | NJS_KEY_USAGE_GENERATE_KEY, NJS_KEY_FORMAT_PKCS8 | - NJS_KEY_FORMAT_SPKI) + NJS_KEY_FORMAT_SPKI | + NJS_KEY_FORMAT_JWK) }, { @@ -217,7 +221,9 @@ static njs_webcrypto_entry_t njs_webcryp NJS_KEY_USAGE_VERIFY | NJS_KEY_USAGE_GENERATE_KEY, NJS_KEY_FORMAT_PKCS8 | - NJS_KEY_FORMAT_SPKI) + NJS_KEY_FORMAT_SPKI | + NJS_KEY_FORMAT_RAW | + NJS_KEY_FORMAT_JWK) }, { @@ -272,9 +278,9 @@ static njs_webcrypto_entry_t njs_webcryp static njs_webcrypto_entry_t njs_webcrypto_curve[] = { - { njs_str("P-256"), NJS_CURVE_P256 }, - { njs_str("P-384"), NJS_CURVE_P384 }, - { njs_str("P-521"), NJS_CURVE_P521 }, + { njs_str("P-256"), NID_X9_62_prime256v1 }, + { njs_str("P-384"), NID_secp384r1 }, + { njs_str("P-521"), NID_secp521r1 }, { njs_null_str, 0 } }; @@ -301,6 +307,51 @@ static njs_webcrypto_entry_t njs_webcryp }; +static njs_webcrypto_entry_t njs_webcrypto_alg_hash[] = { + { njs_str("RS1"), NJS_HASH_SHA1 }, + { njs_str("RS256"), NJS_HASH_SHA256 }, + { njs_str("RS384"), NJS_HASH_SHA384 }, + { njs_str("RS512"), NJS_HASH_SHA512 }, + { njs_str("PS1"), NJS_HASH_SHA1 }, + { njs_str("PS256"), NJS_HASH_SHA256 }, + { njs_str("PS384"), NJS_HASH_SHA384 }, + { njs_str("PS512"), NJS_HASH_SHA512 }, + { njs_str("RSA-OAEP"), NJS_HASH_SHA1 }, + { njs_str("RSA-OAEP-256"), NJS_HASH_SHA256 }, + { njs_str("RSA-OAEP-384"), NJS_HASH_SHA384 }, + { njs_str("RSA-OAEP-512"), NJS_HASH_SHA512 }, + { njs_null_str, 0 } +}; + + +static njs_str_t + njs_webcrypto_alg_name[NJS_ALGORITHM_RSA_OAEP + 1][NJS_HASH_SHA512 + 1] = { + { + njs_null_str, + njs_str("RS1"), + njs_str("RS256"), + njs_str("RS384"), + njs_str("RS512"), + }, + + { + njs_null_str, + njs_str("PS1"), + njs_str("PS256"), + njs_str("PS384"), + njs_str("PS512"), + }, + + { + njs_null_str, + njs_str("RSA-OAEP"), + njs_str("RSA-OAEP-256"), + njs_str("RSA-OAEP-384"), + njs_str("RSA-OAEP-512"), + }, +}; + + static njs_external_t njs_ext_webcrypto_crypto_key[] = { { @@ -504,6 +555,23 @@ njs_module_t njs_webcrypto_module = { }; +static const njs_value_t string_alg = njs_string("alg"); +static const njs_value_t string_d = njs_string("d"); +static const njs_value_t string_dp = njs_string("dp"); +static const njs_value_t string_dq = njs_string("dq"); +static const njs_value_t string_e = njs_string("e"); +static const njs_value_t string_n = njs_string("n"); +static const njs_value_t string_p = njs_string("p"); +static const njs_value_t string_q = njs_string("q"); +static const njs_value_t string_qi = njs_string("qi"); +static const njs_value_t string_x = njs_string("x"); +static const njs_value_t string_y = njs_string("y"); +static const njs_value_t string_ext = njs_string("ext"); +static const njs_value_t string_crv = njs_string("crv"); +static const njs_value_t string_kty = njs_string("kty"); +static const njs_value_t key_ops = njs_string("key_ops"); + + static njs_int_t njs_webcrypto_crypto_key_proto_id; @@ -1639,11 +1707,484 @@ fail: static njs_int_t +njs_export_base64url_bignum(njs_vm_t *vm, njs_value_t *retval, const BIGNUM *v, + size_t size) +{ + njs_str_t src; + u_char buf[512]; + + if (size == 0) { + size = BN_num_bytes(v); + } + + if (njs_bn_bn2binpad(v, &buf[0], size) <= 0) { + return NJS_ERROR; + } + + src.start = buf; + src.length = size; + + return njs_string_base64url(vm, retval, &src); +} + + +static njs_int_t +njs_base64url_bignum_set(njs_vm_t *vm, njs_value_t *jwk, njs_value_t *key, + const BIGNUM *v, size_t size) +{ + njs_int_t ret; + njs_value_t value; + + ret = njs_export_base64url_bignum(vm, &value, v, size); + if (ret != NJS_OK) { + return NJS_ERROR; + } + + return njs_value_property_set(vm, jwk, key, &value); +} + + +static njs_int_t +njs_export_jwk_rsa(njs_vm_t *vm, njs_webcrypto_key_t *key, njs_value_t *retval) +{ + njs_int_t ret; + const RSA *rsa; + njs_str_t *nm; + njs_value_t nvalue, evalue, alg; + const BIGNUM *n_bn, *e_bn, *d_bn, *p_bn, *q_bn, *dp_bn, *dq_bn, *qi_bn; + + static const njs_value_t rsa_str = njs_string("RSA"); + + rsa = njs_pkey_get_rsa_key(key->pkey); + + njs_rsa_get0_key(rsa, &n_bn, &e_bn, &d_bn); + + ret = njs_export_base64url_bignum(vm, &nvalue, n_bn, 0); + if (ret != NJS_OK) { + return NJS_ERROR; + } + + ret = njs_export_base64url_bignum(vm, &evalue, e_bn, 0); + if (ret != NJS_OK) { + return NJS_ERROR; + } + + ret = njs_vm_object_alloc(vm, retval, &string_kty, &rsa_str, &string_n, + &nvalue, &string_e, &evalue, NULL); + if (ret != NJS_OK) { + return NJS_ERROR; + } + + if (key->privat) { + njs_rsa_get0_factors(rsa, &p_bn, &q_bn); + njs_rsa_get0_ctr_params(rsa, &dp_bn, &dq_bn, &qi_bn); + + ret = njs_base64url_bignum_set(vm, retval, njs_value_arg(&string_d), + d_bn, 0); + if (ret != NJS_OK) { + return NJS_ERROR; + } + + ret = njs_base64url_bignum_set(vm, retval, njs_value_arg(&string_p), + p_bn, 0); + if (ret != NJS_OK) { + return NJS_ERROR; + } + + ret = njs_base64url_bignum_set(vm, retval, njs_value_arg(&string_q), + q_bn, 0); + if (ret != NJS_OK) { + return NJS_ERROR; + } + + ret = njs_base64url_bignum_set(vm, retval, njs_value_arg(&string_dp), + dp_bn, 0); + if (ret != NJS_OK) { + return NJS_ERROR; + } + + ret = njs_base64url_bignum_set(vm, retval, njs_value_arg(&string_dq), + dq_bn, 0); + if (ret != NJS_OK) { + return NJS_ERROR; + } + + ret = njs_base64url_bignum_set(vm, retval, njs_value_arg(&string_qi), + qi_bn, 0); + if (ret != NJS_OK) { + return NJS_ERROR; + } + } + + nm = &njs_webcrypto_alg_name[key->alg->type][key->hash]; + + (void) njs_vm_value_string_set(vm, &alg, nm->start, nm->length); + + return njs_value_property_set(vm, retval, njs_value_arg(&string_alg), &alg); +} + + +static njs_int_t +njs_export_jwk_ec(njs_vm_t *vm, njs_webcrypto_key_t *key, njs_value_t *retval) +{ + int nid, group_bits, group_bytes; + BIGNUM *x_bn, *y_bn; + njs_int_t ret; + njs_value_t xvalue, yvalue, dvalue, name; + const EC_KEY *ec; + const BIGNUM *d_bn; + const EC_POINT *pub; + const EC_GROUP *group; + njs_webcrypto_entry_t *e; + + static const njs_value_t ec_str = njs_string("EC"); + + x_bn = NULL; + y_bn = NULL; + d_bn = NULL; + + ec = njs_pkey_get_ec_key(key->pkey); + + pub = EC_KEY_get0_public_key(ec); + group = EC_KEY_get0_group(ec); + + group_bits = EC_GROUP_get_degree(group); + group_bytes = (group_bits / CHAR_BIT) + (7 + (group_bits % CHAR_BIT)) / 8; + + x_bn = BN_new(); + if (x_bn == NULL) { + goto fail; + } + + y_bn = BN_new(); + if (y_bn == NULL) { + goto fail; + } + + if (!njs_ec_point_get_affine_coordinates(group, pub, x_bn, y_bn)) { + njs_webcrypto_error(vm, "EC_POINT_get_affine_coordinates() failed"); + goto fail; + } + + ret = njs_export_base64url_bignum(vm, &xvalue, x_bn, group_bytes); + if (ret != NJS_OK) { + goto fail; + } + + BN_free(x_bn); + x_bn = NULL; + + ret = njs_export_base64url_bignum(vm, &yvalue, y_bn, group_bytes); + if (ret != NJS_OK) { + goto fail; + } + + BN_free(y_bn); + y_bn = NULL; + + nid = EC_GROUP_get_curve_name(group); + + for (e = &njs_webcrypto_curve[0]; e->name.length != 0; e++) { + if ((uintptr_t) nid == e->value) { + (void) njs_vm_value_string_set(vm, &name, e->name.start, + e->name.length); + break; + } + } + + if (e->name.length == 0) { + njs_type_error(vm, "Unsupported JWK EC curve: %s", OBJ_nid2sn(nid)); + goto fail; + } + + ret = njs_vm_object_alloc(vm, retval, &string_kty, &ec_str, &string_x, + &xvalue, &string_y, &yvalue, &string_crv, &name, + NULL); + if (ret != NJS_OK) { + goto fail; + } + + if (key->privat) { + d_bn = EC_KEY_get0_private_key(ec); + + ret = njs_export_base64url_bignum(vm, &dvalue, d_bn, group_bytes); + if (ret != NJS_OK) { + goto fail; + } + + ret = njs_value_property_set(vm, retval, njs_value_arg(&string_d), + &dvalue); + if (ret != NJS_OK) { + goto fail; + } + } + + return NJS_OK; + +fail: + + if (x_bn != NULL) { + BN_free(x_bn); + } + + if (y_bn != NULL) { + BN_free(y_bn); + } + + return NJS_ERROR; +} + + +static njs_int_t +njs_export_raw_ec(njs_vm_t *vm, njs_webcrypto_key_t *key, njs_value_t *retval) +{ + size_t size; + u_char *dst; + const EC_KEY *ec; + const EC_GROUP *group; + const EC_POINT *point; + point_conversion_form_t form; + + njs_assert(key->pkey != NULL); + + if (key->privat) { + njs_type_error(vm, "private key of \"%V\" cannot be exported " + "in \"raw\" format", njs_algorithm_string(key->alg)); + return NJS_ERROR; + } + + ec = njs_pkey_get_ec_key(key->pkey); + + group = EC_KEY_get0_group(ec); + point = EC_KEY_get0_public_key(ec); + form = POINT_CONVERSION_UNCOMPRESSED; + + size = EC_POINT_point2oct(group, point, form, NULL, 0, NULL); + if (njs_slow_path(size == 0)) { + njs_webcrypto_error(vm, "EC_POINT_point2oct() failed"); + return NJS_ERROR; + } + + dst = njs_mp_alloc(njs_vm_memory_pool(vm), size); + if (njs_slow_path(dst == NULL)) { + return NJS_ERROR; + } + + size = EC_POINT_point2oct(group, point, form, dst, size, NULL); + if (njs_slow_path(size == 0)) { + njs_webcrypto_error(vm, "EC_POINT_point2oct() failed"); + return NJS_ERROR; + } + + return njs_vm_value_array_buffer_set(vm, retval, dst, size); +} + + +static njs_int_t +njs_export_jwk_asymmetric(njs_vm_t *vm, njs_webcrypto_key_t *key, + njs_value_t *retval) +{ + njs_int_t ret; + njs_value_t ops, extractable; + + njs_assert(key->pkey != NULL); + + switch (EVP_PKEY_id(key->pkey)) { + case EVP_PKEY_RSA: +#if (OPENSSL_VERSION_NUMBER >= 0x10100001L) + case EVP_PKEY_RSA_PSS: +#endif + ret = njs_export_jwk_rsa(vm, key, retval); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + break; + + case EVP_PKEY_EC: + ret = njs_export_jwk_ec(vm, key, retval); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + break; + + default: + njs_type_error(vm, "provided key cannot be exported as JWK"); + return NJS_ERROR; + } + + ret = njs_key_ops(vm, &ops, key->usage); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + ret = njs_value_property_set(vm, retval, njs_value_arg(&key_ops), &ops); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + njs_value_boolean_set(&extractable, key->extractable); + + return njs_value_property_set(vm, retval, njs_value_arg(&string_ext), + &extractable); +} + + +static njs_int_t njs_ext_export_key(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused) { - njs_internal_error(vm, "\"exportKey\" not implemented"); - return NJS_ERROR; + BIO *bio; + BUF_MEM *mem; + njs_int_t ret; + njs_value_t value; + njs_webcrypto_key_t *key; + PKCS8_PRIV_KEY_INFO *pkcs8; + njs_webcrypto_key_format_t fmt; + + fmt = njs_key_format(vm, njs_arg(args, nargs, 1)); + if (njs_slow_path(fmt == NJS_KEY_FORMAT_UNKNOWN)) { + goto fail; + } + + key = njs_vm_external(vm, njs_webcrypto_crypto_key_proto_id, + njs_arg(args, nargs, 2)); + if (njs_slow_path(key == NULL)) { + njs_type_error(vm, "\"key\" is not a CryptoKey object"); + goto fail; + } + + if (njs_slow_path(!(fmt & key->alg->fmt))) { + njs_type_error(vm, "unsupported key fmt \"%V\" for \"%V\" key", + njs_format_string(fmt), + njs_algorithm_string(key->alg)); + goto fail; + } + + if (njs_slow_path(!key->extractable)) { + njs_type_error(vm, "provided key cannot be extracted"); + goto fail; + } + + switch (fmt) { + case NJS_KEY_FORMAT_JWK: + switch (key->alg->type) { + case NJS_ALGORITHM_RSASSA_PKCS1_v1_5: + case NJS_ALGORITHM_RSA_PSS: + case NJS_ALGORITHM_RSA_OAEP: + case NJS_ALGORITHM_ECDSA: + ret = njs_export_jwk_asymmetric(vm, key, &value); + if (njs_slow_path(ret != NJS_OK)) { + goto fail; + } + + break; + + default: + break; + } + + break; + + case NJS_KEY_FORMAT_PKCS8: + if (!key->privat) { + njs_type_error(vm, "public key of \"%V\" cannot be exported " + "as PKCS8", njs_algorithm_string(key->alg)); + goto fail; + } + + bio = BIO_new(BIO_s_mem()); + if (njs_slow_path(bio == NULL)) { + njs_webcrypto_error(vm, "BIO_new(BIO_s_mem()) failed"); + goto fail; + } + + njs_assert(key->pkey != NULL); + + pkcs8 = EVP_PKEY2PKCS8(key->pkey); + if (njs_slow_path(pkcs8 == NULL)) { + BIO_free(bio); + njs_webcrypto_error(vm, "EVP_PKEY2PKCS8() failed"); + goto fail; + } + + if (!i2d_PKCS8_PRIV_KEY_INFO_bio(bio, pkcs8)) { + BIO_free(bio); + PKCS8_PRIV_KEY_INFO_free(pkcs8); + njs_webcrypto_error(vm, "i2d_PKCS8_PRIV_KEY_INFO_bio() failed"); + goto fail; + } + + BIO_get_mem_ptr(bio, &mem); + + ret = njs_webcrypto_array_buffer(vm, &value, (u_char *) mem->data, + mem->length); + + BIO_free(bio); + PKCS8_PRIV_KEY_INFO_free(pkcs8); + + if (njs_slow_path(ret != NJS_OK)) { + goto fail; + } + + break; + + case NJS_KEY_FORMAT_SPKI: + if (key->privat) { + njs_type_error(vm, "private key of \"%V\" cannot be exported " + "as SPKI", njs_algorithm_string(key->alg)); + goto fail; + } + + bio = BIO_new(BIO_s_mem()); + if (njs_slow_path(bio == NULL)) { + njs_webcrypto_error(vm, "BIO_new(BIO_s_mem()) failed"); + goto fail; + } + + njs_assert(key->pkey != NULL); + + if (!i2d_PUBKEY_bio(bio, key->pkey)) { + BIO_free(bio); + njs_webcrypto_error(vm, "i2d_PUBKEY_bio() failed"); + goto fail; + } + + BIO_get_mem_ptr(bio, &mem); + + ret = njs_webcrypto_array_buffer(vm, &value, (u_char *) mem->data, + mem->length); + + BIO_free(bio); + + if (njs_slow_path(ret != NJS_OK)) { + goto fail; + } + + break; + + case NJS_KEY_FORMAT_RAW: + default: + if (key->alg->type == NJS_ALGORITHM_ECDSA) { + ret = njs_export_raw_ec(vm, key, &value); + if (njs_slow_path(ret != NJS_OK)) { + goto fail; + } + + break; + } + + njs_internal_error(vm, "exporting as \"%V\" fmt is not implemented", + njs_format_string(fmt)); + goto fail; + } + + return njs_webcrypto_result(vm, &value, NJS_OK); + +fail: + + return njs_webcrypto_result(vm, njs_vm_retval(vm), NJS_ERROR); } @@ -1651,8 +2192,711 @@ static njs_int_t njs_ext_generate_key(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused) { - njs_internal_error(vm, "\"generateKey\" not implemented"); - return NJS_ERROR; + int nid; + unsigned usage; + njs_int_t ret; + njs_bool_t extractable; + njs_value_t value, pub, priv, *aobject; + EVP_PKEY_CTX *ctx; + njs_webcrypto_key_t *key, *keypub; + njs_webcrypto_algorithm_t *alg; + + static const njs_value_t string_ml = njs_string("modulusLength"); + static const njs_value_t string_priv = njs_string("privateKey"); + static const njs_value_t string_pub = njs_string("publicKey"); + From xeioex at nginx.com Thu Jan 5 04:42:47 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Jan 2023 04:42:47 +0000 Subject: [njs] WebCrypto: extended support for symmetric keys. Message-ID: details: https://hg.nginx.org/njs/rev/2e3bbe8743af branches: changeset: 2021:2e3bbe8743af user: Dmitry Volyntsev date: Wed Jan 04 18:07:30 2023 -0800 description: WebCrypto: extended support for symmetric keys. The following functionality for HMAC and AES-* keys were added: importKey() supporting 'jwk' format, exportKey() supporting 'jwk' and 'raw' formats, generateKey(). diffstat: external/njs_webcrypto_module.c | 365 +++++++++++++++++++++++++++++++++++++-- test/ts/test.ts | 8 + test/webcrypto/export.t.js | 180 +++++++++++++++++++ test/webcrypto/sign.t.js | 19 ++ ts/njs_webcrypto.d.ts | 12 +- 5 files changed, 555 insertions(+), 29 deletions(-) diffs (806 lines): diff -r 0681bf662222 -r 2e3bbe8743af external/njs_webcrypto_module.c --- a/external/njs_webcrypto_module.c Wed Jan 04 17:49:22 2023 -0800 +++ b/external/njs_webcrypto_module.c Wed Jan 04 18:07:30 2023 -0800 @@ -167,7 +167,8 @@ static njs_webcrypto_entry_t njs_webcryp NJS_KEY_USAGE_WRAP_KEY | NJS_KEY_USAGE_UNWRAP_KEY | NJS_KEY_USAGE_GENERATE_KEY, - NJS_KEY_FORMAT_RAW) + NJS_KEY_FORMAT_RAW | + NJS_KEY_FORMAT_JWK) }, { @@ -178,7 +179,8 @@ static njs_webcrypto_entry_t njs_webcryp NJS_KEY_USAGE_WRAP_KEY | NJS_KEY_USAGE_UNWRAP_KEY | NJS_KEY_USAGE_GENERATE_KEY, - NJS_KEY_FORMAT_RAW) + NJS_KEY_FORMAT_RAW | + NJS_KEY_FORMAT_JWK) }, { @@ -189,7 +191,8 @@ static njs_webcrypto_entry_t njs_webcryp NJS_KEY_USAGE_WRAP_KEY | NJS_KEY_USAGE_UNWRAP_KEY | NJS_KEY_USAGE_GENERATE_KEY, - NJS_KEY_FORMAT_RAW) + NJS_KEY_FORMAT_RAW | + NJS_KEY_FORMAT_JWK) }, { @@ -258,7 +261,8 @@ static njs_webcrypto_entry_t njs_webcryp NJS_KEY_USAGE_GENERATE_KEY | NJS_KEY_USAGE_SIGN | NJS_KEY_USAGE_VERIFY, - NJS_KEY_FORMAT_RAW) + NJS_KEY_FORMAT_RAW | + NJS_KEY_FORMAT_JWK) }, { @@ -325,7 +329,7 @@ static njs_webcrypto_entry_t njs_webcryp static njs_str_t - njs_webcrypto_alg_name[NJS_ALGORITHM_RSA_OAEP + 1][NJS_HASH_SHA512 + 1] = { + njs_webcrypto_alg_name[NJS_ALGORITHM_HMAC + 1][NJS_HASH_SHA512 + 1] = { { njs_null_str, njs_str("RS1"), @@ -349,6 +353,37 @@ static njs_str_t njs_str("RSA-OAEP-384"), njs_str("RSA-OAEP-512"), }, + + { + njs_null_str, + njs_str("HS1"), + njs_str("HS256"), + njs_str("HS384"), + njs_str("HS512"), + }, +}; + +static njs_str_t njs_webcrypto_alg_aes_name[3][3 + 1] = { + { + njs_str("A128GCM"), + njs_str("A192GCM"), + njs_str("A256GCM"), + njs_null_str, + }, + + { + njs_str("A128CTR"), + njs_str("A192CTR"), + njs_str("A256CTR"), + njs_null_str, + }, + + { + njs_str("A128CBC"), + njs_str("A192CBC"), + njs_str("A256CBC"), + njs_null_str, + }, }; @@ -560,6 +595,7 @@ static const njs_value_t string_d = njs static const njs_value_t string_dp = njs_string("dp"); static const njs_value_t string_dq = njs_string("dq"); static const njs_value_t string_e = njs_string("e"); +static const njs_value_t string_k = njs_string("k"); static const njs_value_t string_n = njs_string("n"); static const njs_value_t string_p = njs_string("p"); static const njs_value_t string_q = njs_string("q"); @@ -570,6 +606,7 @@ static const njs_value_t string_ext = n static const njs_value_t string_crv = njs_string("crv"); static const njs_value_t string_kty = njs_string("kty"); static const njs_value_t key_ops = njs_string("key_ops"); +static const njs_value_t string_length = njs_string("length"); static njs_int_t njs_webcrypto_crypto_key_proto_id; @@ -1036,7 +1073,6 @@ njs_cipher_aes_ctr(njs_vm_t *vm, njs_str u_char iv2[16]; static const njs_value_t string_counter = njs_string("counter"); - static const njs_value_t string_length = njs_string("length"); switch (key->raw.length) { case 16: @@ -1356,7 +1392,6 @@ njs_ext_derive(njs_vm_t *vm, njs_value_t static const njs_value_t string_info = njs_string("info"); static const njs_value_t string_salt = njs_string("salt"); - static const njs_value_t string_length = njs_string("length"); static const njs_value_t string_iterations = njs_string("iterations"); aobject = njs_arg(args, nargs, 1); @@ -2032,6 +2067,71 @@ njs_export_jwk_asymmetric(njs_vm_t *vm, static njs_int_t +njs_export_jwk_oct(njs_vm_t *vm, njs_webcrypto_key_t *key, njs_value_t *retval) +{ + njs_int_t ret; + njs_str_t *nm; + njs_value_t k, alg, ops, extractable; + njs_webcrypto_alg_t type; + + static const njs_value_t oct_str = njs_string("oct"); + + njs_assert(key->raw.start != NULL) + + ret = njs_string_base64url(vm, &k, &key->raw); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + type = key->alg->type; + + if (key->alg->type == NJS_ALGORITHM_HMAC) { + nm = &njs_webcrypto_alg_name[type][key->hash]; + (void) njs_vm_value_string_set(vm, &alg, nm->start, nm->length); + + } else { + switch (key->raw.length) { + case 16: + case 24: + case 32: + nm = &njs_webcrypto_alg_aes_name + [type - NJS_ALGORITHM_AES_GCM][(key->raw.length - 16) / 8]; + (void) njs_vm_value_string_set(vm, &alg, nm->start, nm->length); + break; + + default: + njs_value_undefined_set(&alg); + break; + } + } + + ret = njs_key_ops(vm, &ops, key->usage); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + njs_value_boolean_set(&extractable, key->extractable); + + ret = njs_vm_object_alloc(vm, retval, &string_kty, &oct_str, &string_k, + &k, &key_ops, &ops, &string_ext, &extractable, + NULL); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + if (njs_is_defined(&alg)) { + ret = njs_value_property_set(vm, retval, njs_value_arg(&string_alg), + &alg); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + } + + return NJS_OK; +} + + +static njs_int_t njs_ext_export_key(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused) { @@ -2081,6 +2181,17 @@ njs_ext_export_key(njs_vm_t *vm, njs_val break; + case NJS_ALGORITHM_AES_GCM: + case NJS_ALGORITHM_AES_CTR: + case NJS_ALGORITHM_AES_CBC: + case NJS_ALGORITHM_HMAC: + ret = njs_export_jwk_oct(vm, key, &value); + if (njs_slow_path(ret != NJS_OK)) { + goto fail; + } + + break; + default: break; } @@ -2175,9 +2286,13 @@ njs_ext_export_key(njs_vm_t *vm, njs_val break; } - njs_internal_error(vm, "exporting as \"%V\" fmt is not implemented", - njs_format_string(fmt)); - goto fail; + ret = njs_vm_value_array_buffer_set(vm, &value, key->raw.start, + key->raw.length); + if (njs_slow_path(ret != NJS_OK)) { + goto fail; + } + + break; } return njs_webcrypto_result(vm, &value, NJS_OK); @@ -2386,6 +2501,57 @@ njs_ext_generate_key(njs_vm_t *vm, njs_v break; + case NJS_ALGORITHM_AES_GCM: + case NJS_ALGORITHM_AES_CTR: + case NJS_ALGORITHM_AES_CBC: + case NJS_ALGORITHM_HMAC: + + if (alg->type == NJS_ALGORITHM_HMAC) { + ret = njs_algorithm_hash(vm, aobject, &key->hash); + if (njs_slow_path(ret == NJS_ERROR)) { + goto fail; + } + + key->raw.length = EVP_MD_size(njs_algorithm_hash_digest(key->hash)); + + } else { + ret = njs_value_property(vm, aobject, njs_value_arg(&string_length), + &value); + if (njs_slow_path(ret == NJS_ERROR)) { + goto fail; + } + + key->raw.length = njs_number(&value) / 8; + + if (key->raw.length != 16 + && key->raw.length != 24 + && key->raw.length != 32) + { + njs_type_error(vm, "length for \"%V\" key should be one of " + "128, 192, 256", njs_algorithm_string(alg)); + goto fail; + } + } + + key->raw.start = njs_mp_alloc(njs_vm_memory_pool(vm), key->raw.length); + if (njs_slow_path(key->raw.start == NULL)) { + njs_memory_error(vm); + goto fail; + } + + if (RAND_bytes(key->raw.start, key->raw.length) <= 0) { + njs_webcrypto_error(vm, "RAND_bytes() failed"); + goto fail; + } + + ret = njs_vm_external_create(vm, &value, + njs_webcrypto_crypto_key_proto_id, key, 0); + if (njs_slow_path(ret != NJS_OK)) { + goto fail; + } + + break; + default: njs_internal_error(vm, "not implemented generateKey" "algorithm: \"%V\"", njs_algorithm_string(alg)); @@ -2901,6 +3067,124 @@ fail: static njs_int_t +njs_import_jwk_oct(njs_vm_t *vm, njs_value_t *jwk, njs_webcrypto_key_t *key) +{ + size_t size; + unsigned usage; + njs_int_t ret; + njs_str_t *a, alg, b64; + njs_value_t value; + njs_webcrypto_alg_t type; + njs_webcrypto_entry_t *w; + + static njs_webcrypto_entry_t hashes[] = { + { njs_str("HS1"), NJS_HASH_SHA1 }, + { njs_str("HS256"), NJS_HASH_SHA256 }, + { njs_str("HS384"), NJS_HASH_SHA384 }, + { njs_str("HS512"), NJS_HASH_SHA512 }, + { njs_null_str, 0 } + }; + + ret = njs_value_property(vm, jwk, njs_value_arg(&string_k), &value); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + if (!njs_value_is_string(&value)) { + njs_type_error(vm, "Invalid JWK oct key"); + return NJS_ERROR; + } + + njs_string_get(&value, &b64); + + (void) njs_decode_base64url_length(&b64, &key->raw.length); + + key->raw.start = njs_mp_alloc(njs_vm_memory_pool(vm), key->raw.length); + if (njs_slow_path(key->raw.start == NULL)) { + njs_memory_error(vm); + return NJS_ERROR; + } + + njs_decode_base64url(&key->raw, &b64); + + ret = njs_value_property(vm, jwk, njs_value_arg(&string_alg), &value); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + size = 16; + + if (njs_value_is_string(&value)) { + njs_string_get(&value, &alg); + + if (key->alg->type == NJS_ALGORITHM_HMAC) { + for (w = &hashes[0]; w->name.length != 0; w++) { + if (njs_strstr_eq(&alg, &w->name)) { + key->hash = w->value; + goto done; + } + } + + } else { + type = key->alg->type; + a = &njs_webcrypto_alg_aes_name[type - NJS_ALGORITHM_AES_GCM][0]; + for (; a->length != 0; a++) { + if (njs_strstr_eq(&alg, a)) { + goto done; + } + + size += 8; + } + } + + njs_type_error(vm, "unexpected \"alg\" value \"%V\" for JWK key", &alg); + return NJS_ERROR; + } + +done: + + if (key->alg->type != NJS_ALGORITHM_HMAC) { + if (key->raw.length != size) { + njs_type_error(vm, "key size and \"alg\" value \"%V\" mismatch", + &alg); + return NJS_ERROR; + } + } + + ret = njs_value_property(vm, jwk, njs_value_arg(&key_ops), &value); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + if (njs_is_defined(&value)) { + ret = njs_key_usage(vm, &value, &usage); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + if ((key->usage & usage) != key->usage) { + njs_type_error(vm, "Key operations and usage mismatch"); + return NJS_ERROR; + } + } + + if (key->extractable) { + ret = njs_value_property(vm, jwk, njs_value_arg(&string_ext), &value); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + if (njs_is_defined(&value) && !njs_value_bool(&value)) { + njs_type_error(vm, "JWK oct is not extractable"); + return NJS_ERROR; + } + } + + return NJS_OK; +} + + +static njs_int_t njs_ext_import_key(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused) { @@ -3057,6 +3341,12 @@ njs_ext_import_key(njs_vm_t *vm, njs_val goto fail; } + } else if (njs_strstr_eq(&kty, &njs_str_value("oct"))) { + ret = njs_import_jwk_oct(vm, jwk, key); + if (njs_slow_path(ret != NJS_OK)) { + goto fail; + } + } else { njs_type_error(vm, "invalid JWK key type: %V", &kty); goto fail; @@ -3182,34 +3472,54 @@ njs_ext_import_key(njs_vm_t *vm, njs_val break; case NJS_ALGORITHM_HMAC: - ret = njs_algorithm_hash(vm, options, &key->hash); - if (njs_slow_path(ret == NJS_ERROR)) { - goto fail; + if (fmt == NJS_KEY_FORMAT_RAW) { + ret = njs_algorithm_hash(vm, options, &key->hash); + if (njs_slow_path(ret == NJS_ERROR)) { + goto fail; + } + + key->raw = key_data; + + } else { + /* NJS_KEY_FORMAT_JWK. */ + + ret = njs_algorithm_hash(vm, options, &hash); + if (njs_slow_path(ret == NJS_ERROR)) { + goto fail; + } + + if (key->hash != NJS_HASH_UNSET && key->hash != hash) { + njs_type_error(vm, "HMAC JWK hash mismatch"); + goto fail; + } } - key->raw = key_data; break; case NJS_ALGORITHM_AES_GCM: case NJS_ALGORITHM_AES_CTR: case NJS_ALGORITHM_AES_CBC: - switch (key_data.length) { - case 16: - case 24: - case 32: - break; - - default: - njs_type_error(vm, "Invalid key length"); - goto fail; + if (fmt == NJS_KEY_FORMAT_RAW) { + switch (key_data.length) { + case 16: + case 24: + case 32: + break; + + default: + njs_type_error(vm, "AES Invalid key length"); + goto fail; + } + + key->raw = key_data; } - /* Fall through. */ + break; case NJS_ALGORITHM_PBKDF2: case NJS_ALGORITHM_HKDF: + default: key->raw = key_data; - default: break; } @@ -3869,6 +4179,11 @@ njs_key_usage(njs_vm_t *vm, njs_value_t njs_int_t ret; njs_iterator_args_t args; + if (!njs_value_is_object(value)) { + njs_type_error(vm, "\"keyUsages\" argument must be an Array"); + return NJS_ERROR; + } + ret = njs_object_length(vm, value, &length); if (njs_slow_path(ret != NJS_OK)) { return NJS_ERROR; diff -r 0681bf662222 -r 2e3bbe8743af test/ts/test.ts --- a/test/ts/test.ts Wed Jan 04 17:49:22 2023 -0800 +++ b/test/ts/test.ts Wed Jan 04 18:07:30 2023 -0800 @@ -188,6 +188,14 @@ async function crypto_object(keyData: Ar modulusLength: 2048, publicExponent: new Uint8Array([1, 0, 1])}, true, ['sign', 'verify']); + + let hkey = await crypto.subtle.generateKey({name: "HMAC", + hash: "SHA-384"}, + true, ['sign', 'verify']); + + let akey = await crypto.subtle.generateKey({name: "AES-GCM", + length: 256}, + true, ['encrypt', 'decrypt']); } function buffer(b: Buffer) { diff -r 0681bf662222 -r 2e3bbe8743af test/webcrypto/export.t.js --- a/test/webcrypto/export.t.js Wed Jan 04 17:49:22 2023 -0800 +++ b/test/webcrypto/export.t.js Wed Jan 04 18:07:30 2023 -0800 @@ -17,6 +17,12 @@ async function load_key(params) { return params.generate_keys.keys[type]; } + if (params.generate_key) { + return await crypto.subtle.generateKey(params.generate_key.alg, + params.generate_key.extractable, + params.generate_key.usage); + } + return await crypto.subtle.importKey(params.key.fmt, params.key.key, params.key.alg, @@ -43,6 +49,7 @@ async function test(params) { } if (!params.generate_keys + && !params.generate_key && (exp.startsWith && !exp.startsWith("ArrayBuffer:"))) { /* Check that exported key can be imported back. */ @@ -73,6 +80,9 @@ function p(args, default_opts) { case "jwk": key = load_jwk(params.key.key); break; + case "raw": + key = Buffer.from(params.key.key, "base64url"); + break; default: throw Error("Unknown encoding key format"); } @@ -291,8 +301,178 @@ let ec_tsuite = { expected: { kty: "EC", ext: true, key_ops: [ "sign" ], crv: "P-384" } }, ]}; +function validate_hmac_jwk(exp, params) { + let hash = params.generate_key.alg.hash; + let expected_len = Number(hash.slice(2)) / 8 * (4 / 3); + expected_len = Math.round(expected_len); + + validate_property(exp, 'k', expected_len); + + return true; +} + +let hmac_tsuite = { + name: "HMAC exporting", + skip: () => (!has_fs() || !has_webcrypto()), + T: test, + prepare_args: p, + opts: { + key: { fmt: "raw", + key: "c2VjcmV0LUtleTE", + alg: { name: "HMAC", hash: "SHA-256" }, + extractable: true, + usage: [ "sign", "verify" ] }, + export: { fmt: "jwk" }, + expected: { kty: "oct", ext: true }, + }, + + tests: [ + { expected: { key_ops: [ "sign", "verify" ], + alg: "HS256", + k: "c2VjcmV0LUtleTE" } }, + { export: { fmt: "raw" }, + expected: "ArrayBuffer:c2VjcmV0LUtleTE" }, + { export: { fmt: "spki" }, + exception: "TypeError: unsupported key fmt \"spki\" for \"HMAC\"" }, + { export: { fmt: "pksc8" }, + exception: "TypeError: unsupported key fmt \"pksc8\" for \"HMAC\"" }, + { key: { key: "cDBzc3dE", + alg: { hash: "SHA-384" } }, + expected: { key_ops: [ "sign", "verify" ], + alg: "HS384", + k: "cDBzc3dE" } }, + { key: { extractable: false }, + exception: "TypeError: provided key cannot be extracted" }, + + { key: { fmt: "jwk", + key: { kty: "oct", ext: true, k: "c2VjcmV0LUtleTE", alg: "HS256" } }, + expected: { key_ops: [ "sign", "verify" ], + alg: "HS256", + k: "c2VjcmV0LUtleTE" } }, + { key: { fmt: "jwk", + alg: { hash: "SHA-512" }, + key: { kty: "oct", ext: true, k: "c2VjcmV0LUtleTE", alg: "HS512" } }, + expected: { key_ops: [ "sign", "verify" ], + alg: "HS512", + k: "c2VjcmV0LUtleTE" } }, + { key: { fmt: "jwk", + key: { kty: "oct", ext: true, k: "c2VjcmV0LUtleTE", alg: "HS256" }, + alg: { hash: "SHA-384" } }, + exception: "TypeError: HMAC JWK hash mismatch" }, + + { generate_key: { alg: { name: "HMAC", + hash: "SHA-256" }, + extractable: true, + usage: [ "sign", "verify" ] }, + check: validate_hmac_jwk, + expected: { key_ops: [ "sign", "verify" ], alg: "HS256" } }, + { generate_key: { alg: { name: "HMAC", + hash: "SHA-1" }, + extractable: true, + usage: [ "verify" ] }, + check: validate_hmac_jwk, + expected: { key_ops: [ "verify" ], alg: "HS1" } }, + { generate_key: { alg: { name: "HMAC", + hash: "SHA-384" }, + extractable: true, + usage: [ "sign" ] }, + check: validate_hmac_jwk, + expected: { key_ops: [ "sign" ], alg: "HS384" } }, + { generate_key: { alg: { name: "HMAC", + hash: "SHA-512" }, + extractable: true, + usage: [ "sign" ] }, + check: validate_hmac_jwk, + expected: { key_ops: [ "sign" ], alg: "HS512" } }, +]}; + +function validate_aes_jwk(exp, params) { + let expected_len = params.generate_key.alg.length; + expected_len = expected_len / 8 * (4 / 3); + expected_len = Math.round(expected_len); + + validate_property(exp, 'k', expected_len); + + return true; +} + +let aes_tsuite = { + name: "AES exporting", + skip: () => (!has_fs() || !has_webcrypto()), + T: test, + prepare_args: p, + opts: { + key: { fmt: "raw", + key: "ABEiMwARIjMAESIzABEiMw", + alg: { name: "AES-GCM" }, + extractable: true, + usage: [ "encrypt" ] }, + export: { fmt: "jwk" }, + expected: { kty: "oct", ext: true }, + }, + + tests: [ + { expected: { key_ops: [ "encrypt" ], + alg: "A128GCM", + k: "ABEiMwARIjMAESIzABEiMw" } }, + { export: { fmt: "raw" }, + expected: "ArrayBuffer:ABEiMwARIjMAESIzABEiMw" }, + { key: { key: "ABEiMwARIjMAESIzABEiMwARIjMAESIz", + alg: { name: "AES-CBC" }, + usage: [ "decrypt" ] }, + expected: { key_ops: [ "decrypt" ], + alg: "A192CBC", + k: "ABEiMwARIjMAESIzABEiMwARIjMAESIz" } }, + { key: { key: "ABEiMwARIjMAESIzABEiMwARIjMAESIz", + alg: { name: "AES-CBC" }, + usage: [ "decrypt" ] }, + export: { fmt: "raw" }, + expected: "ArrayBuffer:ABEiMwARIjMAESIzABEiMwARIjMAESIz" }, + { key: { key: "ABEiMwARIjMAESIzABEiMwARIjMAESIzABEiMwARIjM", + alg: { name: "AES-CTR" }, + usage: [ "decrypt" ] }, + expected: { key_ops: [ "decrypt" ], + alg: "A256CTR", + k: "ABEiMwARIjMAESIzABEiMwARIjMAESIzABEiMwARIjM" } }, + { key: { key: "ABEiMwARIjMAESIzABEiMwARIjMAESIzABEiMwARIjM", + alg: { name: "AES-CTR" }, + usage: [ "decrypt" ] }, + export: { fmt: "raw" }, + expected: "ArrayBuffer:ABEiMwARIjMAESIzABEiMwARIjMAESIzABEiMwARIjM" }, + + { generate_key: { alg: { name: "AES-GCM", length: 128 }, + extractable: true, + usage: [ "encrypt" ] }, + check: validate_aes_jwk, + expected: { key_ops: [ "encrypt" ], alg: "A128GCM" } }, + { generate_key: { alg: { name: "AES-CTR", length: 192 }, + extractable: true, + usage: [ "decrypt" ] }, + check: validate_aes_jwk, + expected: { key_ops: [ "decrypt" ], alg: "A192CTR" } }, + { generate_key: { alg: { name: "AES-CBC", length: 256 }, + extractable: true, + usage: [ "decrypt" ] }, + check: validate_aes_jwk, + expected: { key_ops: [ "decrypt" ], alg: "A256CBC" } }, + { generate_key: { alg: { name: "AES-GCM", length: 128 }, + extractable: false, + usage: [ "decrypt" ] }, + exception: "TypeError: provided key cannot be extracted" }, + { generate_key: { alg: { name: "AES-GCM" }, + extractable: false, + usage: [ "decrypt" ] }, + exception: "TypeError: length for \"AES-GCM\" key should be one of 128, 192, 256" }, + { generate_key: { alg: { name: "AES-GCM", length: 25 }, + extractable: false, + usage: [ "decrypt" ] }, + exception: "TypeError: length for \"AES-GCM\" key should be one of 128, 192, 256" }, +]}; + run([ rsa_tsuite, ec_tsuite, + hmac_tsuite, + aes_tsuite, ]) .then($DONE, $DONE); diff -r 0681bf662222 -r 2e3bbe8743af test/webcrypto/sign.t.js --- a/test/webcrypto/sign.t.js Wed Jan 04 17:49:22 2023 -0800 +++ b/test/webcrypto/sign.t.js Wed Jan 04 18:07:30 2023 -0800 @@ -180,6 +180,25 @@ let hmac_tsuite = { expected: "0540c587e7ee607fb4fd5e814438ed50f261c244" }, { sign_alg: { name: "ECDSA" }, exception: "TypeError: cannot sign using \"HMAC\" with \"ECDSA\" key" }, + { sign_key: { fmt: "jwk", + key: { kty: "oct", + alg: "HS256", + k: "c2VjcmV0S0VZ" } }, + expected: "76d4f1b22d7544c34e86380c9ab7c756311810dc31e4af3b705045d263db1212" }, + { sign_key: { fmt: "jwk", + key: { kty: "oct", + alg: "HS256", + key_ops: [ "sign" ], + k: "c2VjcmV0S0VZ" } }, + verify: true, + expected: true }, + { sign_key: { fmt: "jwk", + key: { kty: "oct", + alg: "HS256", + key_ops: [ "verify" ], + k: "c2VjcmV0S0VZ" } }, + exception: "TypeError: Key operations and usage mismatch" }, + { verify: true, expected: true }, { verify: true, import_alg: { hash: "SHA-384" }, expected: true }, { verify: true, import_alg: { hash: "SHA-512" }, expected: true }, diff -r 0681bf662222 -r 2e3bbe8743af ts/njs_webcrypto.d.ts --- a/ts/njs_webcrypto.d.ts Wed Jan 04 17:49:22 2023 -0800 +++ b/ts/njs_webcrypto.d.ts Wed Jan 04 18:07:30 2023 -0800 @@ -72,11 +72,14 @@ type ImportAlgorithm = type GenerateAlgorithm = | RsaHashedKeyGenParams - | EcKeyGenParams; + | EcKeyGenParams + | HmacKeyGenParams + | AesKeyGenParams; type JWK = | { kty: "RSA"; } - | { kty: "EC"; }; + | { kty: "EC"; } + | { kty: "oct"; }; type KeyData = | NjsStringOrBuffer @@ -230,7 +233,8 @@ interface SubtleCrypto { key: CryptoKey): Promise; /** - * Generates a keypair for asymmetric algorithms. + * Generates a key for symmetric algorithms or a keypair + * for asymmetric algorithms. * * @since 0.7.10 * @param algorithm Dictionary object defining the type of key to generate @@ -242,7 +246,7 @@ interface SubtleCrypto { */ generateKey(algorithm: GenerateAlgorithm, extractable: boolean, - usage: Array): Promise; + usage: Array): Promise; /** * Generates a digital signature. From xeioex at nginx.com Thu Jan 5 04:42:49 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Jan 2023 04:42:49 +0000 Subject: [njs] WebCrypto: improved njs_bn_bn2binpad() for OpenSSL < 1.1.0. Message-ID: details: https://hg.nginx.org/njs/rev/2a412a132cf2 branches: changeset: 2022:2a412a132cf2 user: Dmitry Volyntsev date: Wed Jan 04 20:39:21 2023 -0800 description: WebCrypto: improved njs_bn_bn2binpad() for OpenSSL < 1.1.0. The patch makes njs_bn_bn2binpad() more similar to BN_bn2binpad(). diffstat: external/njs_openssl.h | 13 ++++++++++++- external/njs_webcrypto_module.c | 4 ---- 2 files changed, 12 insertions(+), 5 deletions(-) diffs (37 lines): diff -r 2e3bbe8743af -r 2a412a132cf2 external/njs_openssl.h --- a/external/njs_openssl.h Wed Jan 04 18:07:30 2023 -0800 +++ b/external/njs_openssl.h Wed Jan 04 20:39:21 2023 -0800 @@ -62,7 +62,18 @@ njs_bn_bn2binpad(const BIGNUM *bn, unsig #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) return BN_bn2binpad(bn, to, tolen); #else - return BN_bn2bin(bn, &to[tolen - BN_num_bytes(bn)]); + int len; + + len = BN_num_bytes(bn); + + if (tolen > len) { + memset(to, 0, tolen - len); + + } else if (tolen < len) { + return -1; + } + + return BN_bn2bin(bn, &to[tolen - len]); #endif } diff -r 2e3bbe8743af -r 2a412a132cf2 external/njs_webcrypto_module.c --- a/external/njs_webcrypto_module.c Wed Jan 04 18:07:30 2023 -0800 +++ b/external/njs_webcrypto_module.c Wed Jan 04 20:39:21 2023 -0800 @@ -3646,10 +3646,6 @@ njs_convert_der_to_p1363(njs_vm_t *vm, E goto fail; } -#if (OPENSSL_VERSION_NUMBER < 0x10100000L) - memset(data, 0, 2 * n); -#endif - if (njs_bn_bn2binpad(ECDSA_SIG_get0_r(ec_sig), data, n) <= 0) { goto fail; } From arut at nginx.com Thu Jan 5 15:31:33 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Thu, 05 Jan 2023 19:31:33 +0400 Subject: [PATCH 0 of 3] HTTP/3 insert count block improvements Message-ID: src/event/quic/ngx_event_quic_streams.c | 14 ++++++++++++-- src/http/v3/ngx_http_v3_request.c | 2 +- src/http/v3/ngx_http_v3_request.c | 6 ++++++ 3 files changed, 19 insertions(+), 3 deletions(-) From arut at nginx.com Thu Jan 5 15:31:34 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Thu, 05 Jan 2023 19:31:34 +0400 Subject: [PATCH 1 of 3] QUIC: set stream error flag on reset In-Reply-To: References: Message-ID: <480300240a4ea0e03e65.1672932694@arut-laptop> src/event/quic/ngx_event_quic_streams.c | 14 ++++++++++++-- 1 files changed, 12 insertions(+), 2 deletions(-) # HG changeset patch # User Roman Arutyunyan # Date 1672932528 -14400 # Thu Jan 05 19:28:48 2023 +0400 # Branch quic # Node ID 480300240a4ea0e03e656039b9498c78b5adc21e # Parent 987bee4363d10895f4bd1a40fc4347c49763e90f QUIC: set stream error flag on reset. Now, when RESET_STREAM is sent or received, or when streams are closed, stream connection error flag is set. Previously, only stream state was changed, which resulted in setting the error flag only after calling recv()/send()/send_chain(). However, there are cases when none of these functions is called, but it's still important to know if the stream is being closed. For example, when an HTTP/3 request stream is blocked on insert count, receiving RESET_STREAM should trigger stream closure, which was not the case. The change also fixes ngx_http_test_reading() and ngx_http_upstream_check_broken_connection() with QUIC streams. diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c --- a/src/event/quic/ngx_event_quic_streams.c +++ b/src/event/quic/ngx_event_quic_streams.c @@ -203,6 +203,9 @@ ngx_quic_close_streams(ngx_connection_t continue; } + sc->read->error = 1; + sc->write->error = 1; + ngx_quic_set_event(sc->read); ngx_quic_set_event(sc->write); @@ -245,6 +248,10 @@ ngx_quic_do_reset_stream(ngx_quic_stream qs->send_state = NGX_QUIC_STREAM_SEND_RESET_SENT; qs->send_final_size = qs->send_offset; + if (qs->connection) { + qs->connection->write->error = 1; + } + pc = qs->parent; qc = ngx_quic_get_connection(pc); @@ -797,7 +804,6 @@ ngx_quic_stream_recv(ngx_connection_t *c || qs->recv_state == NGX_QUIC_STREAM_RECV_RESET_READ) { qs->recv_state = NGX_QUIC_STREAM_RECV_RESET_READ; - rev->error = 1; return NGX_ERROR; } @@ -1375,6 +1381,7 @@ ngx_int_t ngx_quic_handle_reset_stream_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_reset_stream_frame_t *f) { + ngx_event_t *rev; ngx_quic_stream_t *qs; ngx_quic_connection_t *qc; @@ -1431,7 +1438,10 @@ ngx_quic_handle_reset_stream_frame(ngx_c return ngx_quic_close_stream(qs); } - ngx_quic_set_event(qs->connection->read); + rev = qs->connection->read; + rev->error = 1; + + ngx_quic_set_event(rev); return NGX_OK; } From arut at nginx.com Thu Jan 5 15:31:35 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Thu, 05 Jan 2023 19:31:35 +0400 Subject: [PATCH 2 of 3] HTTP/3: trigger 400 (Bad Request) on stream error while blocked In-Reply-To: References: Message-ID: src/http/v3/ngx_http_v3_request.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) # HG changeset patch # User Roman Arutyunyan # Date 1672928146 -14400 # Thu Jan 05 18:15:46 2023 +0400 # Branch quic # Node ID c44c0a06c4c23e599b164a768d6999880411c2e0 # Parent 480300240a4ea0e03e656039b9498c78b5adc21e HTTP/3: trigger 400 (Bad Request) on stream error while blocked. Previously, stream was closed with NGX_HTTP_CLOSE. However, in a similar case when recv() returns eof or error, status 400 is triggered. diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -554,7 +554,7 @@ ngx_http_v3_process_request(ngx_event_t if (rc == NGX_BUSY) { if (rev->error) { - ngx_http_close_request(r, NGX_HTTP_CLOSE); + ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); break; } From arut at nginx.com Thu Jan 5 15:31:36 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Thu, 05 Jan 2023 19:31:36 +0400 Subject: [PATCH 3 of 3] HTTP/3: insert count block timeout In-Reply-To: References: Message-ID: src/http/v3/ngx_http_v3_request.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) # HG changeset patch # User Roman Arutyunyan # Date 1672931002 -14400 # Thu Jan 05 19:03:22 2023 +0400 # Branch quic # Node ID a8f6e9a1d1674915501a8322004d04e6eac245d8 # Parent c44c0a06c4c23e599b164a768d6999880411c2e0 HTTP/3: insert count block timeout. Previously, there was no timeout for a request stream blocked on insert count, which could result in infinite wait. Now client_header_timeout is set when stream is first blocked. diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -558,6 +558,12 @@ ngx_http_v3_process_request(ngx_event_t break; } + if (!rev->timer_set) { + cscf = ngx_http_get_module_srv_conf(r, + ngx_http_core_module); + ngx_add_timer(rev, cscf->client_header_timeout); + } + if (ngx_handle_read_event(rev, 0) != NGX_OK) { ngx_http_close_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); } From arut at nginx.com Fri Jan 6 08:33:04 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Fri, 6 Jan 2023 12:33:04 +0400 Subject: [PATCH 2 of 6] QUIC: handle datagrams directly in ngx_quic_recvmsg() In-Reply-To: <8a7f2c71db202141d169.1670578728@ip-10-1-18-114.eu-central-1.compute.internal> References: <8a7f2c71db202141d169.1670578728@ip-10-1-18-114.eu-central-1.compute.internal> Message-ID: <20230106083304.qjtjijosbylofshs@N00W24XTQX> Hi, Following a discussion with Sergey, here's a new version with a simplified ngx_quic_run(). On Fri, Dec 09, 2022 at 09:38:48AM +0000, Roman Arutyunyan wrote: > # HG changeset patch > # User Roman Arutyunyan > # Date 1670428292 0 > # Wed Dec 07 15:51:32 2022 +0000 > # Branch quic > # Node ID 8a7f2c71db202141d169f3ab292027bfc16ff8ec > # Parent 1038d7300c29eea02b47eac3f205e293b1e55f5b > QUIC: handle datagrams directly in ngx_quic_recvmsg(). > > Previously, ngx_quic_recvmsg() called client connection's read event handler > to emulate normal event processing. Further, the read event handler handled > the datagram by calling ngx_quic_handle_datagram(). > > Now ngx_quic_handle_datagram() is called directly from ngx_quic_recvmsg(), > which simplifies the code. [..] -- Roman Arutyunyan -------------- next part -------------- # HG changeset patch # User Roman Arutyunyan # Date 1672752567 -14400 # Tue Jan 03 17:29:27 2023 +0400 # Branch quic # Node ID aaa2a3831eefe4315dfb8a9be7178c79ff67f163 # Parent a8f6e9a1d1674915501a8322004d04e6eac245d8 QUIC: handle datagrams directly in ngx_quic_recvmsg(). Previously, ngx_quic_recvmsg() called client connection's read event handler to emulate normal event processing. Further, the read event handler handled the datagram by calling ngx_quic_handle_datagram(). Now ngx_quic_handle_datagram() is called directly from ngx_quic_recvmsg(), which simplifies the code. diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -17,8 +17,6 @@ static ngx_int_t ngx_quic_handle_statele static void ngx_quic_input_handler(ngx_event_t *rev); static void ngx_quic_close_handler(ngx_event_t *ev); -static ngx_int_t ngx_quic_handle_datagram(ngx_connection_t *c, ngx_buf_t *b, - ngx_quic_conf_t *conf); static ngx_int_t ngx_quic_handle_packet(ngx_connection_t *c, ngx_quic_conf_t *conf, ngx_quic_header_t *pkt); static ngx_int_t ngx_quic_handle_payload(ngx_connection_t *c, @@ -201,8 +199,7 @@ ngx_quic_apply_transport_params(ngx_conn void ngx_quic_run(ngx_connection_t *c, ngx_quic_conf_t *conf) { - ngx_int_t rc; - ngx_quic_connection_t *qc; + ngx_int_t rc; ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic run"); @@ -211,16 +208,6 @@ ngx_quic_run(ngx_connection_t *c, ngx_qu ngx_quic_close_connection(c, rc); return; } - - /* quic connection is now created */ - qc = ngx_quic_get_connection(c); - - ngx_add_timer(c->read, qc->tp.max_idle_timeout); - ngx_quic_connstate_dbg(c); - - c->read->handler = ngx_quic_input_handler; - - return; } @@ -340,6 +327,8 @@ ngx_quic_new_connection(ngx_connection_t c->idle = 1; ngx_reusable_connection(c, 1); + c->read->handler = ngx_quic_input_handler; + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic connection created"); @@ -397,8 +386,6 @@ ngx_quic_handle_stateless_reset(ngx_conn static void ngx_quic_input_handler(ngx_event_t *rev) { - ngx_int_t rc; - ngx_buf_t *b; ngx_connection_t *c; ngx_quic_connection_t *qc; @@ -432,29 +419,6 @@ ngx_quic_input_handler(ngx_event_t *rev) return; } - - b = c->udp->buffer; - if (b == NULL) { - return; - } - - rc = ngx_quic_handle_datagram(c, b, NULL); - - if (rc == NGX_ERROR) { - ngx_quic_close_connection(c, NGX_ERROR); - return; - } - - if (rc == NGX_DONE) { - return; - } - - /* rc == NGX_OK */ - - qc->send_timer_set = 0; - ngx_add_timer(rev, qc->tp.max_idle_timeout); - - ngx_quic_connstate_dbg(c); } @@ -654,7 +618,7 @@ ngx_quic_close_handler(ngx_event_t *ev) } -static ngx_int_t +ngx_int_t ngx_quic_handle_datagram(ngx_connection_t *c, ngx_buf_t *b, ngx_quic_conf_t *conf) { @@ -753,6 +717,11 @@ ngx_quic_handle_datagram(ngx_connection_ qc->error_reason = "QUIC flood detected"; return NGX_ERROR; } + + qc->send_timer_set = 0; + ngx_add_timer(c->read, qc->tp.max_idle_timeout); + + ngx_quic_connstate_dbg(c); } return NGX_OK; diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -260,6 +260,8 @@ struct ngx_quic_connection_s { }; +ngx_int_t ngx_quic_handle_datagram(ngx_connection_t *c, ngx_buf_t *b, + ngx_quic_conf_t *conf); ngx_int_t ngx_quic_apply_transport_params(ngx_connection_t *c, ngx_quic_tp_t *ctp); void ngx_quic_discard_ctx(ngx_connection_t *c, diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -264,12 +264,6 @@ ngx_quic_set_path(ngx_connection_t *c, n len = pkt->raw->last - pkt->raw->start; - if (c->udp->buffer == NULL) { - /* first ever packet in connection, path already exists */ - path = qc->path; - goto update; - } - probe = NULL; for (q = ngx_queue_head(&qc->paths); diff --git a/src/event/quic/ngx_event_quic_socket.c b/src/event/quic/ngx_event_quic_socket.c --- a/src/event/quic/ngx_event_quic_socket.c +++ b/src/event/quic/ngx_event_quic_socket.c @@ -61,6 +61,9 @@ ngx_quic_open_sockets(ngx_connection_t * } ngx_memcpy(qc->tp.initial_scid.data, qsock->sid.id, qsock->sid.len); + ngx_memcpy(&qsock->sockaddr.sockaddr, c->sockaddr, c->socklen); + qsock->socklen = c->socklen; + /* for all packets except first, this is set at udp layer */ c->udp = &qsock->udp; diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c --- a/src/event/quic/ngx_event_quic_udp.c +++ b/src/event/quic/ngx_event_quic_udp.c @@ -186,21 +186,11 @@ ngx_quic_recvmsg(ngx_event_t *ev) ngx_memcpy(&qsock->sockaddr.sockaddr, sockaddr, socklen); qsock->socklen = socklen; - c->udp->buffer = &buf; - - rev = c->read; - rev->ready = 1; - rev->active = 0; - - rev->handler(rev); - - if (c->udp) { - c->udp->buffer = NULL; + if (ngx_quic_handle_datagram(c, &buf, NULL) == NGX_ERROR) { + ngx_quic_close_connection(c, NGX_ERROR); + return; } - rev->ready = 0; - rev->active = 1; - goto next; } From pluknet at nginx.com Fri Jan 6 11:17:49 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Fri, 6 Jan 2023 15:17:49 +0400 Subject: [PATCH 1 of 3] QUIC: set stream error flag on reset In-Reply-To: <480300240a4ea0e03e65.1672932694@arut-laptop> References: <480300240a4ea0e03e65.1672932694@arut-laptop> Message-ID: <83CD80CB-F9CC-4E56-AD25-C428D3FD51FD@nginx.com> > On 5 Jan 2023, at 19:31, Roman Arutyunyan wrote: > > src/event/quic/ngx_event_quic_streams.c | 14 ++++++++++++-- > 1 files changed, 12 insertions(+), 2 deletions(-) > > > # HG changeset patch > # User Roman Arutyunyan > # Date 1672932528 -14400 > # Thu Jan 05 19:28:48 2023 +0400 > # Branch quic > # Node ID 480300240a4ea0e03e656039b9498c78b5adc21e > # Parent 987bee4363d10895f4bd1a40fc4347c49763e90f > QUIC: set stream error flag on reset. > > Now, when RESET_STREAM is sent or received, or when streams are closed, stream > connection error flag is set. Previously, only stream state was changed, which > resulted in setting the error flag only after calling > recv()/send()/send_chain(). However, there are cases when none of these > functions is called, but it's still important to know if the stream is being > closed. For example, when an HTTP/3 request stream is blocked on insert count, > receiving RESET_STREAM should trigger stream closure, which was not the case. To make stream closure work when receiving RESET_STREAM, for a stream that blocks on insert count, the read event should be active, so it is posted. It could be made active with ngx_handle_read_event(), but since rev->ready remains set, this prevents from setting rev->active = 1. > > The change also fixes ngx_http_test_reading() and > ngx_http_upstream_check_broken_connection() with QUIC streams. > > diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c > --- a/src/event/quic/ngx_event_quic_streams.c > +++ b/src/event/quic/ngx_event_quic_streams.c > @@ -203,6 +203,9 @@ ngx_quic_close_streams(ngx_connection_t > continue; > } > > + sc->read->error = 1; > + sc->write->error = 1; > + > ngx_quic_set_event(sc->read); > ngx_quic_set_event(sc->write); > > @@ -245,6 +248,10 @@ ngx_quic_do_reset_stream(ngx_quic_stream > qs->send_state = NGX_QUIC_STREAM_SEND_RESET_SENT; > qs->send_final_size = qs->send_offset; > > + if (qs->connection) { > + qs->connection->write->error = 1; > + } > + > pc = qs->parent; > qc = ngx_quic_get_connection(pc); > > @@ -797,7 +804,6 @@ ngx_quic_stream_recv(ngx_connection_t *c > || qs->recv_state == NGX_QUIC_STREAM_RECV_RESET_READ) > { > qs->recv_state = NGX_QUIC_STREAM_RECV_RESET_READ; > - rev->error = 1; > return NGX_ERROR; > } > > @@ -1375,6 +1381,7 @@ ngx_int_t > ngx_quic_handle_reset_stream_frame(ngx_connection_t *c, > ngx_quic_header_t *pkt, ngx_quic_reset_stream_frame_t *f) > { > + ngx_event_t *rev; > ngx_quic_stream_t *qs; > ngx_quic_connection_t *qc; > > @@ -1431,7 +1438,10 @@ ngx_quic_handle_reset_stream_frame(ngx_c > return ngx_quic_close_stream(qs); > } > > - ngx_quic_set_event(qs->connection->read); > + rev = qs->connection->read; > + rev->error = 1; > + > + ngx_quic_set_event(rev); > > return NGX_OK; > } -- Sergey Kandaurov From xeioex at nginx.com Sat Jan 7 00:52:51 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Sat, 07 Jan 2023 00:52:51 +0000 Subject: [njs] Fixed unitialized value usage warning. Message-ID: details: https://hg.nginx.org/njs/rev/1a5d22feadc2 branches: changeset: 2023:1a5d22feadc2 user: Dmitry Volyntsev date: Thu Jan 05 22:05:03 2023 -0800 description: Fixed unitialized value usage warning. Found by Coverity (CID 1518908). diffstat: external/njs_webcrypto_module.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diffs (33 lines): diff -r 2a412a132cf2 -r 1a5d22feadc2 external/njs_webcrypto_module.c --- a/external/njs_webcrypto_module.c Wed Jan 04 20:39:21 2023 -0800 +++ b/external/njs_webcrypto_module.c Thu Jan 05 22:05:03 2023 -0800 @@ -3212,6 +3212,8 @@ njs_ext_import_key(njs_vm_t *vm, njs_val njs_webcrypto_key_format_t fmt; pkey = NULL; + key_data.start = NULL; + key_data.length = 0; fmt = njs_key_format(vm, njs_arg(args, nargs, 1)); if (njs_slow_path(fmt == NJS_KEY_FORMAT_UNKNOWN)) { @@ -3266,11 +3268,9 @@ njs_ext_import_key(njs_vm_t *vm, njs_val * key->hash = NJS_HASH_UNSET; */ - start = key_data.start; - switch (fmt) { case NJS_KEY_FORMAT_PKCS8: - bio = njs_bio_new_mem_buf(start, key_data.length); + bio = njs_bio_new_mem_buf(key_data.start, key_data.length); if (njs_slow_path(bio == NULL)) { njs_webcrypto_error(vm, "BIO_new_mem_buf() failed"); goto fail; @@ -3299,6 +3299,7 @@ njs_ext_import_key(njs_vm_t *vm, njs_val break; case NJS_KEY_FORMAT_SPKI: + start = key_data.start; pkey = d2i_PUBKEY(NULL, &start, key_data.length); if (njs_slow_path(pkey == NULL)) { njs_webcrypto_error(vm, "d2i_PUBKEY() failed"); From xeioex at nginx.com Sat Jan 7 00:52:53 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Sat, 07 Jan 2023 00:52:53 +0000 Subject: [njs] Fixed Object.values() and Object.entries() with external objects. Message-ID: details: https://hg.nginx.org/njs/rev/a98b63e87688 branches: changeset: 2024:a98b63e87688 user: Dmitry Volyntsev date: Fri Jan 06 16:50:46 2023 -0800 description: Fixed Object.values() and Object.entries() with external objects. This closes #606 issue on Github. diffstat: src/njs_value.c | 73 ++++++++++++++++++++++++++++++++++++++++++----- src/test/njs_unit_test.c | 9 +++++ 2 files changed, 73 insertions(+), 9 deletions(-) diffs (112 lines): diff -r 1a5d22feadc2 -r a98b63e87688 src/njs_value.c --- a/src/njs_value.c Thu Jan 05 22:05:03 2023 -0800 +++ b/src/njs_value.c Fri Jan 06 16:50:46 2023 -0800 @@ -231,25 +231,80 @@ njs_array_t * njs_value_own_enumerate(njs_vm_t *vm, njs_value_t *value, njs_object_enum_t kind, njs_object_enum_type_t type, njs_bool_t all) { - njs_int_t ret; - njs_value_t keys; + njs_int_t ret, len; + njs_array_t *values, *entry; + njs_value_t keys, *k, *dst, *end; njs_object_value_t obj_val; njs_exotic_slots_t *slots; if (njs_is_object(value)) { - if (kind == NJS_ENUM_KEYS && (type & NJS_ENUM_STRING)) { - slots = njs_object_slots(value); - if (slots != NULL && slots->keys != NULL) { - ret = slots->keys(vm, value, &keys); + slots = njs_object_slots(value); + if (slots == NULL || slots->keys == NULL) { + return njs_object_own_enumerate(vm, njs_object(value), kind, type, + all); + } + + ret = slots->keys(vm, value, &keys); + if (njs_slow_path(ret != NJS_OK)) { + return NULL; + } + + switch (kind) { + case NJS_ENUM_KEYS: + return njs_array(&keys); + + case NJS_ENUM_VALUES: + len = njs_array_len(&keys); + k = njs_array(&keys)->start; + end = k + len; + + values = njs_array_alloc(vm, 1, len, 0); + if (njs_slow_path(values == NULL)) { + return NULL; + } + + dst = values->start; + + while (k < end) { + ret = njs_value_property(vm, value, k++, dst++); + if (njs_slow_path(ret != NJS_OK)) { + return NULL; + } + } + + return values; + + case NJS_ENUM_BOTH: + default: + len = njs_array_len(&keys); + k = njs_array(&keys)->start; + end = k + len; + + values = njs_array_alloc(vm, 1, len, 0); + if (njs_slow_path(values == NULL)) { + return NULL; + } + + dst = values->start; + + while (k < end) { + entry = njs_array_alloc(vm, 1, 2, 0); + if (njs_slow_path(entry == NULL)) { + return NULL; + } + + ret = njs_value_property(vm, value, k, &entry->start[1]); if (njs_slow_path(ret != NJS_OK)) { return NULL; } - return njs_array(&keys); + njs_value_assign(&entry->start[0], k++); + + njs_set_array(dst++, entry); } + + return values; } - - return njs_object_own_enumerate(vm, njs_object(value), kind, type, all); } if (value->type != NJS_STRING) { diff -r 1a5d22feadc2 -r a98b63e87688 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Thu Jan 05 22:05:03 2023 -0800 +++ b/src/test/njs_unit_test.c Fri Jan 06 16:50:46 2023 -0800 @@ -21712,6 +21712,15 @@ static njs_unit_test_t njs_externals_te { njs_str("njs.dump($r).startsWith('External')"), njs_str("true") }, + { njs_str("Object.keys($r.header)"), + njs_str("01,02,03") }, + + { njs_str("Object.values($r.header)"), + njs_str("01|АБВ,02|АБВ,03|АБВ") }, + + { njs_str("njs.dump(Object.entries($r.header))"), + njs_str("[['01','01|АБВ'],['02','02|АБВ'],['03','03|АБВ']]") }, + { njs_str("njs.dump($r.header)"), njs_str("Header {01:'01|АБВ',02:'02|АБВ',03:'03|АБВ'}") }, From jordanc.carter at outlook.com Sat Jan 7 06:04:09 2023 From: jordanc.carter at outlook.com (J Carter) Date: Sat, 7 Jan 2023 06:04:09 +0000 Subject: [PATCH] Dynamic rate limiting for limit_req module In-Reply-To: References: Message-ID: Hello, Please find below my revised patch with style and code bug fixes included. The only change in behavior of note is that invalid rate values are now simply ignored in the same fashion an empty key is, and the use of the parsing of complex value being disabled if a non variable is used as the rate. A brief overview of the issue and how I've resolved it to match the behavior of the current best solution for multiple rates. > "The fundamental problem with dynamic rates in limit_req, which is > a leaky bucket limiter, is that the rate is a property which > applies to multiple requests as accumulated in the bucket (and > this is basically why it is configured in the limit_req_zone, and > not in limit_req), while the dynamically calculated rate is a > property of the last request. > > This can easily result in counter-intuitive behaviour. > For example, consider 5 requests with 1 second interval, assuming > burst 2, and rate being 1r/m or 100r/s depending on some request > properties: > > - 1st request, rate 1r/m, request is accepted > - 2st request, rate 1r/m, request is accepted > - 3rd request, rate 1r/m, request is rejected, since there is no >    room in the bucket > - 4th request, rate 100r/s, request is accepted > - 5th request, rate 1r/m, request is accepted (unexpectedly) > > Note that the 5th request is accepted, while it is mostly > equivalent to the 3rd request, and one could expect it to be > rejected.  But it happens to to be accepted because the 4th > request "cleared" the bucket." No additional logic was required to fix this. Simply appending the $rate variable to the $key is sufficient for a user to avoid unwanted excess value changes with extreme rate jumps (as seen above). Instead of:   limit_req_zone $binary_remote_addr zone=one:10m rate=$rate; this:               limit_req_zone $binary_remote_addr$rate zone=one:10m rate=$rate; This ensures that a change in the rate variable's value starts new accounting of excess (as a new state is created to reflect this new key) - this follows the behavior previously given in this chain for a static set of rates. As an additional benefit, the syntax is more compact and readable within the nginx configuration at the cost of slight overhead in memory. (the solution previously given for multiple defined/static rates) >     ... >     map $traffic_tier $limit_free { >         free          $binary_remote_addr; >     } > >     map $traffic_tier $limit_basic { >         basic         $binary_remote_addr; >     } > >     map $traffic_tier $limit_premium { >         premium       $binary_remote_addr; >     } > >     limit_req_zone $limit_free zone=free:10m rate=1r/m; >     limit_req_zone $limit_basic zone=basic:10m rate=2r/m; >     limit_req_zone $limit_premium zone=premium:10m rate=1r/s; > >     limit_req zone=free (burst=x nodelay); >     limit_req zone=basic (burst=x nodelay); >     limit_req zone=premium (burst=x nodelay); to ... map $traffic_tier $rate {     free             1r/m;     basic           2r/m;     premium    1r/s; } limit_req_zone $binary_remote_addr$rate zone=one:10m rate=$rate; limit_req zone=one burst=x nodelay; (With the appended $rate after $key being optional - if your rates don't flip flop to extremes omitting it will save memory and allow more states) The is merely a side benefit - the main purpose of the patch is being able to obtain a rate limit in the from a string (of some description) from a request, or any variable that is populated in time for the limit_req module to use directly as a rate limit value. The approach detailed above works for that too. debian at debian:~/projects/nginx-merc/nginx-tests$ prove limit_req* limit_req2.t ......... ok limit_req_delay.t .... ok limit_req_dry_run.t .. ok limit_req.t .......... ok All tests successful. Files=4, Tests=40,  5 wallclock secs ( 0.05 usr  0.01 sys +  0.48 cusr  0.12 csys =  0.66 CPU) Result: PASS # HG changeset patch # User jordanc.carter at outlook.com # Date 1673064770 0 #      Sat Jan 07 04:12:50 2023 +0000 # Branch dynamic-rate-limiting # Node ID f80741fb734e0a4f83f2f96436ff300c4f3125aa # Parent  07b0bee87f32be91a33210bc06973e07c4c1dac9 Variables support for limit_req_zone's rate diff -r 07b0bee87f32 -r f80741fb734e src/http/modules/ngx_http_limit_req_module.c --- a/src/http/modules/ngx_http_limit_req_module.c      Wed Dec 21 14:53:27 2022 +0300 +++ b/src/http/modules/ngx_http_limit_req_module.c      Sat Jan 07 04:12:50 2023 +0000 @@ -26,6 +26,7 @@      /* integer value, 1 corresponds to 0.001 r/s */      ngx_uint_t                   excess;      ngx_uint_t                   count; +    ngx_uint_t                   rate;      u_char                       data[1];  } ngx_http_limit_req_node_t; @@ -41,6 +42,7 @@      ngx_http_limit_req_shctx_t  *sh;      ngx_slab_pool_t             *shpool;      /* integer value, 1 corresponds to 0.001 r/s */ +    ngx_http_complex_value_t     cvr;      ngx_uint_t                   rate;      ngx_http_complex_value_t     key;      ngx_http_limit_req_node_t   *node; @@ -66,9 +68,9 @@  static void ngx_http_limit_req_delay(ngx_http_request_t *r);  static ngx_int_t ngx_http_limit_req_lookup(ngx_http_limit_req_limit_t *limit, -    ngx_uint_t hash, ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account); +    ngx_uint_t hash, ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account, ngx_uint_t rate);  static ngx_msec_t ngx_http_limit_req_account(ngx_http_limit_req_limit_t *limits, -    ngx_uint_t n, ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit); +    ngx_uint_t n, ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit, ngx_uint_t rate);  static void ngx_http_limit_req_unlock(ngx_http_limit_req_limit_t *limits,      ngx_uint_t n);  static void ngx_http_limit_req_expire(ngx_http_limit_req_ctx_t *ctx, @@ -195,10 +197,12 @@  ngx_http_limit_req_handler(ngx_http_request_t *r)  {      uint32_t                     hash; -    ngx_str_t                    key; +    ngx_str_t                    key, s;      ngx_int_t                    rc;      ngx_uint_t                   n, excess; +    ngx_uint_t                   scale, rate;      ngx_msec_t                   delay; +    u_char                      *p;      ngx_http_limit_req_ctx_t    *ctx;      ngx_http_limit_req_conf_t   *lrcf;      ngx_http_limit_req_limit_t  *limit, *limits; @@ -211,6 +215,7 @@      limits = lrcf->limits.elts;      excess = 0; +    rate = 0;      rc = NGX_DECLINED; @@ -243,10 +248,44 @@          hash = ngx_crc32_short(key.data, key.len); +        if (ctx->rate == 0) { + +            scale = 1; + +            if (ngx_http_complex_value(r, &ctx->cvr, &s) != NGX_OK) { +                ngx_http_limit_req_unlock(limits, n); +                return NGX_HTTP_INTERNAL_SERVER_ERROR; +            } + +            if (s.len < 9) { +                continue; +            } + +            rate = (ngx_uint_t) ngx_atoi(s.data + 5, s.len - 8); + +            if (rate == 0) { +                continue; +            } + +            p = s.data + s.len - 3; + +            if (ngx_strncmp(p, "r/m", 3) == 0) { +                scale = 60; + +            } else if (ngx_strncmp(p, "r/s", 3) != 0) { +                continue; +            } + +            rate = rate * 1000 / scale; + +        } else { +            rate = ctx->rate; +        } +          ngx_shmtx_lock(&ctx->shpool->mutex);          rc = ngx_http_limit_req_lookup(limit, hash, &key, &excess, -                                       (n == lrcf->limits.nelts - 1)); +                                       (n == lrcf->limits.nelts - 1), rate);          ngx_shmtx_unlock(&ctx->shpool->mutex); @@ -291,7 +330,7 @@          excess = 0;      } -    delay = ngx_http_limit_req_account(limits, n, &excess, &limit); +    delay = ngx_http_limit_req_account(limits, n, &excess, &limit, rate);      if (!delay) {          r->main->limit_req_status = NGX_HTTP_LIMIT_REQ_PASSED; @@ -403,7 +442,7 @@  static ngx_int_t  ngx_http_limit_req_lookup(ngx_http_limit_req_limit_t *limit, ngx_uint_t hash, -    ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account) +    ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account, ngx_uint_t rate)  {      size_t                      size;      ngx_int_t                   rc, excess; @@ -451,7 +490,9 @@                  ms = 0;              } -            excess = lr->excess - ctx->rate * ms / 1000 + 1000; +            excess = lr->excess - rate * ms / 1000 + 1000; + +            lr->rate = rate;              if (excess < 0) {                  excess = 0; @@ -510,6 +551,7 @@      lr->len = (u_short) key->len;      lr->excess = 0; +    lr->rate = rate;      ngx_memcpy(lr->data, key->data, key->len); @@ -534,7 +576,7 @@  static ngx_msec_t  ngx_http_limit_req_account(ngx_http_limit_req_limit_t *limits, ngx_uint_t n, -    ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit) +    ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit, ngx_uint_t rate)  {      ngx_int_t                   excess;      ngx_msec_t                  now, delay, max_delay; @@ -548,8 +590,7 @@          max_delay = 0;      } else { -        ctx = (*limit)->shm_zone->data; -        max_delay = (excess - (*limit)->delay) * 1000 / ctx->rate; +        max_delay = (excess - (*limit)->delay) * 1000 / rate;      }      while (n--) { @@ -572,7 +613,7 @@              ms = 0;          } -        excess = lr->excess - ctx->rate * ms / 1000 + 1000; +        excess = lr->excess - lr->rate * ms / 1000 + 1000;          if (excess < 0) {              excess = 0; @@ -593,7 +634,7 @@              continue;          } -        delay = (excess - limits[n].delay) * 1000 / ctx->rate; +        delay = (excess - limits[n].delay) * 1000 / lr->rate;          if (delay > max_delay) {              max_delay = delay; @@ -676,7 +717,7 @@                  return;              } -            excess = lr->excess - ctx->rate * ms / 1000; +            excess = lr->excess - lr->rate * ms / 1000;              if (excess > 0) {                  return; @@ -860,7 +901,7 @@      }      size = 0; -    rate = 1; +    rate = 0;      scale = 1;      name.len = 0; @@ -902,6 +943,20 @@          if (ngx_strncmp(value[i].data, "rate=", 5) == 0) { +            ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); + +            ccv.cf = cf; +            ccv.value = &value[i]; +            ccv.complex_value = &ctx->cvr; + +            if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { +                return NGX_CONF_ERROR; +            } + +            if (ctx->cvr.lengths != NULL) { +                continue; +            } +              len = value[i].len;              p = value[i].data + len - 3; From jordanc.carter at outlook.com Sun Jan 8 03:01:45 2023 From: jordanc.carter at outlook.com (J Carter) Date: Sun, 8 Jan 2023 03:01:45 +0000 Subject: [PATCH] Dynamic rate limiting for limit_req module In-Reply-To: <7451de9a-8f0a-5f96-09ac-2370289e9cde@outlook.com> References: <7451de9a-8f0a-5f96-09ac-2370289e9cde@outlook.com> Message-ID: *with the underflow from atoi error handled. # HG changeset patch # User jordanc.carter at outlook.com # Date 1673146061 0 #      Sun Jan 08 02:47:41 2023 +0000 # Branch dynamic-rate-limiting # Node ID 72cabfd1397057a598af8925a335b80b7eba2da1 # Parent  07b0bee87f32be91a33210bc06973e07c4c1dac9 Variables support for limit_req_zone's rate diff -r 07b0bee87f32 -r 72cabfd13970 src/http/modules/ngx_http_limit_req_module.c --- a/src/http/modules/ngx_http_limit_req_module.c      Wed Dec 21 14:53:27 2022 +0300 +++ b/src/http/modules/ngx_http_limit_req_module.c      Sun Jan 08 02:47:41 2023 +0000 @@ -26,6 +26,7 @@      /* integer value, 1 corresponds to 0.001 r/s */      ngx_uint_t                   excess;      ngx_uint_t                   count; +    ngx_uint_t                   rate;      u_char                       data[1];  } ngx_http_limit_req_node_t; @@ -41,6 +42,7 @@      ngx_http_limit_req_shctx_t  *sh;      ngx_slab_pool_t             *shpool;      /* integer value, 1 corresponds to 0.001 r/s */ +    ngx_http_complex_value_t     cvr;      ngx_uint_t                   rate;      ngx_http_complex_value_t     key;      ngx_http_limit_req_node_t   *node; @@ -66,9 +68,9 @@  static void ngx_http_limit_req_delay(ngx_http_request_t *r);  static ngx_int_t ngx_http_limit_req_lookup(ngx_http_limit_req_limit_t *limit, -    ngx_uint_t hash, ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account); +    ngx_uint_t hash, ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account, ngx_uint_t rate);  static ngx_msec_t ngx_http_limit_req_account(ngx_http_limit_req_limit_t *limits, -    ngx_uint_t n, ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit); +    ngx_uint_t n, ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit, ngx_uint_t rate);  static void ngx_http_limit_req_unlock(ngx_http_limit_req_limit_t *limits,      ngx_uint_t n);  static void ngx_http_limit_req_expire(ngx_http_limit_req_ctx_t *ctx, @@ -195,10 +197,12 @@  ngx_http_limit_req_handler(ngx_http_request_t *r)  {      uint32_t                     hash; -    ngx_str_t                    key; +    ngx_str_t                    key, s;      ngx_int_t                    rc;      ngx_uint_t                   n, excess; +    ngx_uint_t                   scale, rate;      ngx_msec_t                   delay; +    u_char                      *p;      ngx_http_limit_req_ctx_t    *ctx;      ngx_http_limit_req_conf_t   *lrcf;      ngx_http_limit_req_limit_t  *limit, *limits; @@ -211,6 +215,7 @@      limits = lrcf->limits.elts;      excess = 0; +    rate = 0;      rc = NGX_DECLINED; @@ -243,10 +248,44 @@          hash = ngx_crc32_short(key.data, key.len); +        if (ctx->rate == 0) { + +            scale = 1; + +            if (ngx_http_complex_value(r, &ctx->cvr, &s) != NGX_OK) { +                ngx_http_limit_req_unlock(limits, n); +                return NGX_HTTP_INTERNAL_SERVER_ERROR; +            } + +            if (s.len < 9) { +                continue; +            } + +            rate = (ngx_uint_t) ngx_atoi(s.data + 5, s.len - 8); + +            if (rate == 0 || rate == (ngx_uint_t) NGX_ERROR) { +                continue; +            } + +            p = s.data + s.len - 3; + +            if (ngx_strncmp(p, "r/m", 3) == 0) { +                scale = 60; + +            } else if (ngx_strncmp(p, "r/s", 3) != 0) { +                continue; +            } + +            rate = rate * 1000 / scale; + +        } else { +            rate = ctx->rate; +        } +          ngx_shmtx_lock(&ctx->shpool->mutex);          rc = ngx_http_limit_req_lookup(limit, hash, &key, &excess, -                                       (n == lrcf->limits.nelts - 1)); +                                       (n == lrcf->limits.nelts - 1), rate);          ngx_shmtx_unlock(&ctx->shpool->mutex); @@ -291,7 +330,7 @@          excess = 0;      } -    delay = ngx_http_limit_req_account(limits, n, &excess, &limit); +    delay = ngx_http_limit_req_account(limits, n, &excess, &limit, rate);      if (!delay) {          r->main->limit_req_status = NGX_HTTP_LIMIT_REQ_PASSED; @@ -403,7 +442,7 @@  static ngx_int_t  ngx_http_limit_req_lookup(ngx_http_limit_req_limit_t *limit, ngx_uint_t hash, -    ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account) +    ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account, ngx_uint_t rate)  {      size_t                      size;      ngx_int_t                   rc, excess; @@ -451,7 +490,9 @@                  ms = 0;              } -            excess = lr->excess - ctx->rate * ms / 1000 + 1000; +            excess = lr->excess - rate * ms / 1000 + 1000; + +            lr->rate = rate;              if (excess < 0) {                  excess = 0; @@ -510,6 +551,7 @@      lr->len = (u_short) key->len;      lr->excess = 0; +    lr->rate = rate;      ngx_memcpy(lr->data, key->data, key->len); @@ -534,7 +576,7 @@  static ngx_msec_t  ngx_http_limit_req_account(ngx_http_limit_req_limit_t *limits, ngx_uint_t n, -    ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit) +    ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit, ngx_uint_t rate)  {      ngx_int_t                   excess;      ngx_msec_t                  now, delay, max_delay; @@ -548,8 +590,7 @@          max_delay = 0;      } else { -        ctx = (*limit)->shm_zone->data; -        max_delay = (excess - (*limit)->delay) * 1000 / ctx->rate; +        max_delay = (excess - (*limit)->delay) * 1000 / rate;      }      while (n--) { @@ -572,7 +613,7 @@              ms = 0;          } -        excess = lr->excess - ctx->rate * ms / 1000 + 1000; +        excess = lr->excess - lr->rate * ms / 1000 + 1000;          if (excess < 0) {              excess = 0; @@ -593,7 +634,7 @@              continue;          } -        delay = (excess - limits[n].delay) * 1000 / ctx->rate; +        delay = (excess - limits[n].delay) * 1000 / lr->rate;          if (delay > max_delay) {              max_delay = delay; @@ -676,7 +717,7 @@                  return;              } -            excess = lr->excess - ctx->rate * ms / 1000; +            excess = lr->excess - lr->rate * ms / 1000;              if (excess > 0) {                  return; @@ -860,7 +901,7 @@      }      size = 0; -    rate = 1; +    rate = 0;      scale = 1;      name.len = 0; @@ -902,6 +943,20 @@          if (ngx_strncmp(value[i].data, "rate=", 5) == 0) { +            ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); + +            ccv.cf = cf; +            ccv.value = &value[i]; +            ccv.complex_value = &ctx->cvr; + +            if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { +                return NGX_CONF_ERROR; +            } + +            if (ctx->cvr.lengths != NULL) { +                continue; +            } +              len = value[i].len;              p = value[i].data + len - 3; On 07/01/2023 06:04, J Carter wrote: > Hello, > > Please find below my revised patch with style and code bug fixes > included. The only change in behavior of note is that invalid rate > values are now simply ignored in the same fashion an empty key is, and > the use of the parsing of complex value being disabled if a non > variable is used as the rate. > > A brief overview of the issue and how I've resolved it to match the > behavior of the current best solution for multiple rates. > > > "The fundamental problem with dynamic rates in limit_req, which is > > a leaky bucket limiter, is that the rate is a property which > > applies to multiple requests as accumulated in the bucket (and > > this is basically why it is configured in the limit_req_zone, and > > not in limit_req), while the dynamically calculated rate is a > > property of the last request. > > > > This can easily result in counter-intuitive behaviour. > > For example, consider 5 requests with 1 second interval, assuming > > burst 2, and rate being 1r/m or 100r/s depending on some request > > properties: > > > > - 1st request, rate 1r/m, request is accepted > > - 2st request, rate 1r/m, request is accepted > > - 3rd request, rate 1r/m, request is rejected, since there is no > >    room in the bucket > > - 4th request, rate 100r/s, request is accepted > > - 5th request, rate 1r/m, request is accepted (unexpectedly) > > > > Note that the 5th request is accepted, while it is mostly > > equivalent to the 3rd request, and one could expect it to be > > rejected.  But it happens to to be accepted because the 4th > > request "cleared" the bucket." > > No additional logic was required to fix this. Simply appending the > $rate variable to the $key is sufficient for a user to avoid unwanted > excess value changes with extreme rate jumps (as seen above). > > Instead of:   limit_req_zone $binary_remote_addr zone=one:10m rate=$rate; > > this:               limit_req_zone $binary_remote_addr$rate > zone=one:10m rate=$rate; > > This ensures that a change in the rate variable's value starts new > accounting of excess (as a new state is created to reflect this new > key) - this follows the behavior previously given in this chain for a > static set of rates. As an additional benefit, the syntax is more > compact and readable within the nginx configuration at the cost of > slight overhead in memory. > > > (the solution previously given for multiple defined/static rates) > > >     ... > >     map $traffic_tier $limit_free { > >         free          $binary_remote_addr; > >     } > > > >     map $traffic_tier $limit_basic { > >         basic         $binary_remote_addr; > >     } > > > >     map $traffic_tier $limit_premium { > >         premium       $binary_remote_addr; > >     } > > > >     limit_req_zone $limit_free zone=free:10m rate=1r/m; > >     limit_req_zone $limit_basic zone=basic:10m rate=2r/m; > >     limit_req_zone $limit_premium zone=premium:10m rate=1r/s; > > > >     limit_req zone=free (burst=x nodelay); > >     limit_req zone=basic (burst=x nodelay); > >     limit_req zone=premium (burst=x nodelay); > > to > > ... > > map $traffic_tier $rate { >     free             1r/m; >     basic           2r/m; >     premium    1r/s; > } > > limit_req_zone $binary_remote_addr$rate zone=one:10m rate=$rate; > limit_req zone=one burst=x nodelay; > > (With the appended $rate after $key being optional - if your rates > don't flip flop to extremes omitting it will save memory and allow > more states) > > The is merely a side benefit - the main purpose of the patch is being > able to obtain a rate limit in the from a string (of some description) > from a request, or any variable that is populated in time for the > limit_req module to use directly as a rate limit value. The approach > detailed above works for that too. > > > debian at debian:~/projects/nginx-merc/nginx-tests$ prove limit_req* > limit_req2.t ......... ok > limit_req_delay.t .... ok > limit_req_dry_run.t .. ok > limit_req.t .......... ok > All tests successful. > Files=4, Tests=40,  5 wallclock secs ( 0.05 usr  0.01 sys +  0.48 > cusr  0.12 csys =  0.66 CPU) > Result: PASS > > # HG changeset patch > # User jordanc.carter at outlook.com > # Date 1673064770 0 > #      Sat Jan 07 04:12:50 2023 +0000 > # Branch dynamic-rate-limiting > # Node ID f80741fb734e0a4f83f2f96436ff300c4f3125aa > # Parent  07b0bee87f32be91a33210bc06973e07c4c1dac9 > Variables support for limit_req_zone's rate > > diff -r 07b0bee87f32 -r f80741fb734e > src/http/modules/ngx_http_limit_req_module.c > --- a/src/http/modules/ngx_http_limit_req_module.c      Wed Dec 21 > 14:53:27 2022 +0300 > +++ b/src/http/modules/ngx_http_limit_req_module.c      Sat Jan 07 > 04:12:50 2023 +0000 > @@ -26,6 +26,7 @@ >      /* integer value, 1 corresponds to 0.001 r/s */ >      ngx_uint_t                   excess; >      ngx_uint_t                   count; > +    ngx_uint_t                   rate; >      u_char                       data[1]; >  } ngx_http_limit_req_node_t; > > @@ -41,6 +42,7 @@ >      ngx_http_limit_req_shctx_t  *sh; >      ngx_slab_pool_t             *shpool; >      /* integer value, 1 corresponds to 0.001 r/s */ > +    ngx_http_complex_value_t     cvr; >      ngx_uint_t                   rate; >      ngx_http_complex_value_t     key; >      ngx_http_limit_req_node_t   *node; > @@ -66,9 +68,9 @@ > >  static void ngx_http_limit_req_delay(ngx_http_request_t *r); >  static ngx_int_t ngx_http_limit_req_lookup(ngx_http_limit_req_limit_t > *limit, > -    ngx_uint_t hash, ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t > account); > +    ngx_uint_t hash, ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t > account, ngx_uint_t rate); >  static ngx_msec_t > ngx_http_limit_req_account(ngx_http_limit_req_limit_t *limits, > -    ngx_uint_t n, ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit); > +    ngx_uint_t n, ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit, > ngx_uint_t rate); >  static void ngx_http_limit_req_unlock(ngx_http_limit_req_limit_t > *limits, >      ngx_uint_t n); >  static void ngx_http_limit_req_expire(ngx_http_limit_req_ctx_t *ctx, > @@ -195,10 +197,12 @@ >  ngx_http_limit_req_handler(ngx_http_request_t *r) >  { >      uint32_t                     hash; > -    ngx_str_t                    key; > +    ngx_str_t                    key, s; >      ngx_int_t                    rc; >      ngx_uint_t                   n, excess; > +    ngx_uint_t                   scale, rate; >      ngx_msec_t                   delay; > +    u_char                      *p; >      ngx_http_limit_req_ctx_t    *ctx; >      ngx_http_limit_req_conf_t   *lrcf; >      ngx_http_limit_req_limit_t  *limit, *limits; > @@ -211,6 +215,7 @@ >      limits = lrcf->limits.elts; > >      excess = 0; > +    rate = 0; > >      rc = NGX_DECLINED; > > @@ -243,10 +248,44 @@ > >          hash = ngx_crc32_short(key.data, key.len); > > +        if (ctx->rate == 0) { > + > +            scale = 1; > + > +            if (ngx_http_complex_value(r, &ctx->cvr, &s) != NGX_OK) { > +                ngx_http_limit_req_unlock(limits, n); > +                return NGX_HTTP_INTERNAL_SERVER_ERROR; > +            } > + > +            if (s.len < 9) { > +                continue; > +            } > + > +            rate = (ngx_uint_t) ngx_atoi(s.data + 5, s.len - 8); > + > +            if (rate == 0) { > +                continue; > +            } > + > +            p = s.data + s.len - 3; > + > +            if (ngx_strncmp(p, "r/m", 3) == 0) { > +                scale = 60; > + > +            } else if (ngx_strncmp(p, "r/s", 3) != 0) { > +                continue; > +            } > + > +            rate = rate * 1000 / scale; > + > +        } else { > +            rate = ctx->rate; > +        } > + >          ngx_shmtx_lock(&ctx->shpool->mutex); > >          rc = ngx_http_limit_req_lookup(limit, hash, &key, &excess, > -                                       (n == lrcf->limits.nelts - 1)); > +                                       (n == lrcf->limits.nelts - 1), > rate); > >          ngx_shmtx_unlock(&ctx->shpool->mutex); > > @@ -291,7 +330,7 @@ >          excess = 0; >      } > > -    delay = ngx_http_limit_req_account(limits, n, &excess, &limit); > +    delay = ngx_http_limit_req_account(limits, n, &excess, &limit, > rate); > >      if (!delay) { >          r->main->limit_req_status = NGX_HTTP_LIMIT_REQ_PASSED; > @@ -403,7 +442,7 @@ > >  static ngx_int_t >  ngx_http_limit_req_lookup(ngx_http_limit_req_limit_t *limit, > ngx_uint_t hash, > -    ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account) > +    ngx_str_t *key, ngx_uint_t *ep, ngx_uint_t account, ngx_uint_t rate) >  { >      size_t                      size; >      ngx_int_t                   rc, excess; > @@ -451,7 +490,9 @@ >                  ms = 0; >              } > > -            excess = lr->excess - ctx->rate * ms / 1000 + 1000; > +            excess = lr->excess - rate * ms / 1000 + 1000; > + > +            lr->rate = rate; > >              if (excess < 0) { >                  excess = 0; > @@ -510,6 +551,7 @@ > >      lr->len = (u_short) key->len; >      lr->excess = 0; > +    lr->rate = rate; > >      ngx_memcpy(lr->data, key->data, key->len); > > @@ -534,7 +576,7 @@ > >  static ngx_msec_t >  ngx_http_limit_req_account(ngx_http_limit_req_limit_t *limits, > ngx_uint_t n, > -    ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit) > +    ngx_uint_t *ep, ngx_http_limit_req_limit_t **limit, ngx_uint_t rate) >  { >      ngx_int_t                   excess; >      ngx_msec_t                  now, delay, max_delay; > @@ -548,8 +590,7 @@ >          max_delay = 0; > >      } else { > -        ctx = (*limit)->shm_zone->data; > -        max_delay = (excess - (*limit)->delay) * 1000 / ctx->rate; > +        max_delay = (excess - (*limit)->delay) * 1000 / rate; >      } > >      while (n--) { > @@ -572,7 +613,7 @@ >              ms = 0; >          } > > -        excess = lr->excess - ctx->rate * ms / 1000 + 1000; > +        excess = lr->excess - lr->rate * ms / 1000 + 1000; > >          if (excess < 0) { >              excess = 0; > @@ -593,7 +634,7 @@ >              continue; >          } > > -        delay = (excess - limits[n].delay) * 1000 / ctx->rate; > +        delay = (excess - limits[n].delay) * 1000 / lr->rate; > >          if (delay > max_delay) { >              max_delay = delay; > @@ -676,7 +717,7 @@ >                  return; >              } > > -            excess = lr->excess - ctx->rate * ms / 1000; > +            excess = lr->excess - lr->rate * ms / 1000; > >              if (excess > 0) { >                  return; > @@ -860,7 +901,7 @@ >      } > >      size = 0; > -    rate = 1; > +    rate = 0; >      scale = 1; >      name.len = 0; > > @@ -902,6 +943,20 @@ > >          if (ngx_strncmp(value[i].data, "rate=", 5) == 0) { > > +            ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); > + > +            ccv.cf = cf; > +            ccv.value = &value[i]; > +            ccv.complex_value = &ctx->cvr; > + > +            if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { > +                return NGX_CONF_ERROR; > +            } > + > +            if (ctx->cvr.lengths != NULL) { > +                continue; > +            } > + >              len = value[i].len; >              p = value[i].data + len - 3; > > > > -------------- next part -------------- An HTML attachment was scrubbed... URL: From arut at nginx.com Mon Jan 9 11:13:16 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 9 Jan 2023 15:13:16 +0400 Subject: [PATCH] QUIC: OpenSSL compatibility layer In-Reply-To: <64a365dcb52503e91d91.1671606452@arut-laptop> References: <64a365dcb52503e91d91.1671606452@arut-laptop> Message-ID: <20230109111316.yfqj24unoubibihz@N00W24XTQX> Hi, On Wed, Dec 21, 2022 at 11:07:32AM +0400, Roman Arutyunyan wrote: > # HG changeset patch > # User Roman Arutyunyan > # Date 1671606197 -14400 > # Wed Dec 21 11:03:17 2022 +0400 > # Branch quic > # Node ID 64a365dcb52503e91d91c2084f56d072301e65f9 > # Parent b87a0dbc1150f415def5bc1e1f00d02b33519026 > QUIC: OpenSSL compatibility layer. > > The change allows to compile QUIC with OpenSSL, which lacks QUIC API. > > The layer is enabled by "--with-openssl-quic-compat" configure option. Here's an updated version of the patch. The compatibility layer is now enabled by default if BoringSSL QUIC API is not available. 0-RTT is not supported in this version and will be done separately. [..] -- Roman Arutyunyan -------------- next part -------------- # HG changeset patch # User Roman Arutyunyan # Date 1673262402 -14400 # Mon Jan 09 15:06:42 2023 +0400 # Branch quic # Node ID 4e5dfe13c84fe50bec639f1b7dcc81604378a42b # Parent aaa2a3831eefe4315dfb8a9be7178c79ff67f163 QUIC: OpenSSL compatibility layer. The change allows to compile QUIC with OpenSSL which lacks BoringSSL QUIC API. This implementation does not support 0-RTT. diff --git a/README b/README --- a/README +++ b/README @@ -53,7 +53,7 @@ 1. Introduction 2. Installing - A library that provides QUIC support is required to build nginx, there + A library that provides QUIC support is recommended to build nginx, there are several of those available on the market: + BoringSSL [4] + LibreSSL [5] @@ -85,6 +85,10 @@ 2. Installing --with-cc-opt="-I../libressl/build/include" \ --with-ld-opt="-L../libressl/build/lib" + Alternatively, nginx can be configured with OpenSSL compatibility + layer, which emulates BoringSSL QUIC API for OpenSSL. This mode is + enabled by default if native QUIC support is not detected. + When configuring nginx, it's possible to enable QUIC and HTTP/3 using the following new configuration options: diff --git a/auto/lib/openssl/conf b/auto/lib/openssl/conf --- a/auto/lib/openssl/conf +++ b/auto/lib/openssl/conf @@ -10,6 +10,7 @@ if [ $OPENSSL != NONE ]; then if [ $USE_OPENSSL_QUIC = YES ]; then have=NGX_QUIC . auto/have + have=NGX_QUIC_OPENSSL_COMPAT . auto/have fi case "$CC" in @@ -124,6 +125,45 @@ else CORE_INCS="$CORE_INCS $ngx_feature_path" CORE_LIBS="$CORE_LIBS $ngx_feature_libs" OPENSSL=YES + + if [ $USE_OPENSSL_QUIC = YES ]; then + + ngx_feature="OpenSSL QUIC support" + ngx_feature_name="NGX_OPENSSL_QUIC" + ngx_feature_run=no + ngx_feature_incs="#include " + ngx_feature_test="SSL_set_quic_method(NULL, NULL)" + . auto/feature + + if [ $ngx_found = no ]; then + + ngx_feature="OpenSSL QUIC compatibility" + ngx_feature_name="NGX_QUIC_OPENSSL_COMPAT" + ngx_feature_run=no + ngx_feature_incs="#include " + ngx_feature_test=" + SSL_set_max_early_data(NULL, TLS1_3_VERSION); + SSL_CTX_set_msg_callback(NULL, NULL); + SSL_CTX_set_keylog_callback(NULL, NULL); + SSL_CTX_add_custom_ext(NULL, 0, 0, NULL, NULL, + NULL, NULL, NULL)" + . auto/feature + fi + + if [ $ngx_found = no ]; then +cat << END + +$0: error: certain modules require OpenSSL QUIC support. +You can either do not enable the modules, or install the OpenSSL library with +QUIC support into the system, or build the OpenSSL library with QUIC support +statically from the source with nginx by using --with-openssl= option. + +END + exit 1 + fi + + have=NGX_QUIC . auto/have + fi fi fi @@ -139,29 +179,4 @@ with nginx by using --with-openssl= option. - -END - exit 1 - fi - fi fi diff --git a/auto/modules b/auto/modules --- a/auto/modules +++ b/auto/modules @@ -1342,7 +1342,8 @@ if [ $USE_OPENSSL_QUIC = YES ]; then src/event/quic/ngx_event_quic_tokens.h \ src/event/quic/ngx_event_quic_ack.h \ src/event/quic/ngx_event_quic_output.h \ - src/event/quic/ngx_event_quic_socket.h" + src/event/quic/ngx_event_quic_socket.h \ + src/event/quic/ngx_event_quic_openssl_compat.h" ngx_module_srcs="src/event/quic/ngx_event_quic.c \ src/event/quic/ngx_event_quic_udp.c \ src/event/quic/ngx_event_quic_transport.c \ @@ -1355,7 +1356,8 @@ if [ $USE_OPENSSL_QUIC = YES ]; then src/event/quic/ngx_event_quic_tokens.c \ src/event/quic/ngx_event_quic_ack.c \ src/event/quic/ngx_event_quic_output.c \ - src/event/quic/ngx_event_quic_socket.c" + src/event/quic/ngx_event_quic_socket.c \ + src/event/quic/ngx_event_quic_openssl_compat.c" ngx_module_libs= ngx_module_link=YES diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -9,6 +9,10 @@ #include #include +#if (NGX_QUIC_OPENSSL_COMPAT) +#include +#endif + #define NGX_SSL_PASSWORD_BUFFER_SIZE 4096 @@ -392,6 +396,10 @@ ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_ SSL_CTX_set_info_callback(ssl->ctx, ngx_ssl_info_callback); +#if (NGX_QUIC_OPENSSL_COMPAT) + ngx_quic_compat_init(ssl->ctx); +#endif + return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -24,6 +24,9 @@ typedef struct ngx_quic_send_ctx_s ng typedef struct ngx_quic_socket_s ngx_quic_socket_t; typedef struct ngx_quic_path_s ngx_quic_path_t; typedef struct ngx_quic_keys_s ngx_quic_keys_t; +#if (NGX_QUIC_OPENSSL_COMPAT) +typedef struct ngx_quic_compat_s ngx_quic_compat_t; +#endif #include #include @@ -36,6 +39,9 @@ typedef struct ngx_quic_keys_s ng #include #include #include +#if (NGX_QUIC_OPENSSL_COMPAT) +#include +#endif /* RFC 9002, 6.2.2. Handshakes and New Paths: kInitialRtt */ @@ -236,6 +242,10 @@ struct ngx_quic_connection_s { ngx_uint_t nshadowbufs; #endif +#if (NGX_QUIC_OPENSSL_COMPAT) + ngx_quic_compat_t *compat; +#endif + ngx_quic_streams_t streams; ngx_quic_congestion_t congestion; diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c new file mode 100644 --- /dev/null +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -0,0 +1,656 @@ + +/* + * Copyright (C) Nginx, Inc. + */ + + +#include +#include +#include +#include + + +#if (NGX_QUIC_OPENSSL_COMPAT) + +#define NGX_QUIC_COMPAT_RECORD_SIZE 1024 + +#define NGX_QUIC_COMPAT_SSL_TP_EXT 0x39 + +#define NGX_QUIC_COMPAT_CLIENT_EARLY "CLIENT_EARLY_TRAFFIC_SECRET" +#define NGX_QUIC_COMPAT_CLIENT_HANDSHAKE "CLIENT_HANDSHAKE_TRAFFIC_SECRET" +#define NGX_QUIC_COMPAT_SERVER_HANDSHAKE "SERVER_HANDSHAKE_TRAFFIC_SECRET" +#define NGX_QUIC_COMPAT_CLIENT_APPLICATION "CLIENT_TRAFFIC_SECRET_0" +#define NGX_QUIC_COMPAT_SERVER_APPLICATION "SERVER_TRAFFIC_SECRET_0" + + +typedef struct { + ngx_quic_secret_t secret; + ngx_uint_t cipher; +} ngx_quic_compat_keys_t; + + +typedef struct { + ngx_log_t *log; + + u_char type; + ngx_str_t payload; + uint64_t number; + ngx_quic_compat_keys_t *keys; + + enum ssl_encryption_level_t level; +} ngx_quic_compat_record_t; + + +struct ngx_quic_compat_s { + const SSL_QUIC_METHOD *method; + + enum ssl_encryption_level_t write_level; + enum ssl_encryption_level_t read_level; + + uint64_t read_record; + ngx_quic_compat_keys_t keys; + + ngx_str_t tp; + ngx_str_t ctp; +}; + + +static void ngx_quic_compat_keylog_callback(const SSL *ssl, const char *line); +static ngx_int_t ngx_quic_compat_set_encryption_secret(ngx_log_t *log, + ngx_quic_compat_keys_t *keys, enum ssl_encryption_level_t level, + const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len); +static int ngx_quic_compat_add_transport_params_callback(SSL *ssl, + unsigned int ext_type, unsigned int context, const unsigned char **out, + size_t *outlen, X509 *x, size_t chainidx, int *al, void *add_arg); +static int ngx_quic_compat_parse_transport_params_callback(SSL *ssl, + unsigned int ext_type, unsigned int context, const unsigned char *in, + size_t inlen, X509 *x, size_t chainidx, int *al, void *parse_arg); +static void ngx_quic_compat_message_callback(int write_p, int version, + int content_type, const void *buf, size_t len, SSL *ssl, void *arg); +static size_t ngx_quic_compat_create_header(ngx_quic_compat_record_t *rec, + u_char *out, ngx_uint_t plain); +static ngx_int_t ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec, + ngx_str_t *res); + + +ngx_int_t +ngx_quic_compat_init(SSL_CTX *ctx) +{ + SSL_CTX_set_keylog_callback(ctx, ngx_quic_compat_keylog_callback); + + if (SSL_CTX_add_custom_ext(ctx, NGX_QUIC_COMPAT_SSL_TP_EXT, + SSL_EXT_CLIENT_HELLO + |SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS, + ngx_quic_compat_add_transport_params_callback, + NULL, + NULL, + ngx_quic_compat_parse_transport_params_callback, + NULL) + == 0) + { + return NGX_ERROR; + } + + return NGX_OK; +} + + +static void +ngx_quic_compat_keylog_callback(const SSL *ssl, const char *line) +{ + u_char ch, *p, *start, value; + size_t n; + ngx_uint_t write; + const SSL_CIPHER *cipher; + ngx_quic_compat_t *com; + ngx_connection_t *c; + ngx_quic_connection_t *qc; + enum ssl_encryption_level_t level; + u_char secret[EVP_MAX_MD_SIZE]; + + c = ngx_ssl_get_connection(ssl); + if (c->type != SOCK_DGRAM) { + return; + } + + p = (u_char *) line; + + for (start = p; *p && *p != ' '; p++); + + n = p - start; + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic compat secret %*s", n, start); + + if (n == sizeof(NGX_QUIC_COMPAT_CLIENT_EARLY) - 1 + && ngx_strncmp(start, NGX_QUIC_COMPAT_CLIENT_EARLY, n) == 0) + { + level = ssl_encryption_early_data; + write = 0; + + } else if (n == sizeof(NGX_QUIC_COMPAT_CLIENT_HANDSHAKE) - 1 + && ngx_strncmp(start, NGX_QUIC_COMPAT_CLIENT_HANDSHAKE, n) == 0) + { + level = ssl_encryption_handshake; + write = 0; + + } else if (n == sizeof(NGX_QUIC_COMPAT_SERVER_HANDSHAKE) - 1 + && ngx_strncmp(start, NGX_QUIC_COMPAT_SERVER_HANDSHAKE, n) == 0) + { + level = ssl_encryption_handshake; + write = 1; + + } else if (n == sizeof(NGX_QUIC_COMPAT_CLIENT_APPLICATION) - 1 + && ngx_strncmp(start, NGX_QUIC_COMPAT_CLIENT_APPLICATION, n) + == 0) + { + level = ssl_encryption_application; + write = 0; + + } else if (n == sizeof(NGX_QUIC_COMPAT_SERVER_APPLICATION) - 1 + && ngx_strncmp(start, NGX_QUIC_COMPAT_SERVER_APPLICATION, n) + == 0) + { + level = ssl_encryption_application; + write = 1; + + } else { + return; + } + + if (*p++ == '\0') { + return; + } + + for ( /* void */ ; *p && *p != ' '; p++); + + if (*p++ == '\0') { + return; + } + + for (n = 0, start = p; *p; p++) { + ch = *p; + + if (ch >= '0' && ch <= '9') { + value = ch - '0'; + goto next; + } + + ch = (u_char) (ch | 0x20); + + if (ch >= 'a' && ch <= 'f') { + value = ch - 'a' + 10; + goto next; + } + + ngx_log_error(NGX_LOG_EMERG, c->log, 0, + "invalid OpenSSL QUIC secret format"); + + return; + + next: + + if ((p - start) % 2) { + secret[n++] += value; + + } else { + if (n >= EVP_MAX_MD_SIZE) { + ngx_log_error(NGX_LOG_EMERG, c->log, 0, + "too big OpenSSL QUIC secret"); + return; + } + + secret[n] = (value << 4); + } + } + + qc = ngx_quic_get_connection(c); + com = qc->compat; + cipher = SSL_get_current_cipher(ssl); + + if (write) { + com->method->set_write_secret((SSL *) ssl, level, cipher, secret, n); + com->write_level = level; + + } else { + com->method->set_read_secret((SSL *) ssl, level, cipher, secret, n); + com->read_level = level; + com->read_record = 0; + + (void) ngx_quic_compat_set_encryption_secret(c->log, &com->keys, level, + cipher, secret, n); + } +} + + +static ngx_int_t +ngx_quic_compat_set_encryption_secret(ngx_log_t *log, + ngx_quic_compat_keys_t *keys, enum ssl_encryption_level_t level, + const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len) +{ + ngx_int_t key_len; + ngx_str_t secret_str; + ngx_uint_t i; + ngx_quic_hkdf_t seq[2]; + ngx_quic_secret_t *peer_secret; + ngx_quic_ciphers_t ciphers; + + peer_secret = &keys->secret; + + keys->cipher = SSL_CIPHER_get_id(cipher); + + key_len = ngx_quic_ciphers(keys->cipher, &ciphers, level); + + if (key_len == NGX_ERROR) { + ngx_ssl_error(NGX_LOG_INFO, log, 0, "unexpected cipher"); + return NGX_ERROR; + } + + if (sizeof(peer_secret->secret.data) < secret_len) { + ngx_log_error(NGX_LOG_ALERT, log, 0, + "unexpected secret len: %uz", secret_len); + return NGX_ERROR; + } + + peer_secret->secret.len = secret_len; + ngx_memcpy(peer_secret->secret.data, secret, secret_len); + + peer_secret->key.len = key_len; + peer_secret->iv.len = NGX_QUIC_IV_LEN; + + secret_str.len = secret_len; + secret_str.data = (u_char *) secret; + + ngx_quic_hkdf_set(&seq[0], "tls13 key", &peer_secret->key, &secret_str); + ngx_quic_hkdf_set(&seq[1], "tls13 iv", &peer_secret->iv, &secret_str); + + for (i = 0; i < (sizeof(seq) / sizeof(seq[0])); i++) { + if (ngx_quic_hkdf_expand(&seq[i], ciphers.d, log) != NGX_OK) { + return NGX_ERROR; + } + } + + return NGX_OK; +} + + +static int +ngx_quic_compat_add_transport_params_callback(SSL *ssl, unsigned int ext_type, + unsigned int context, const unsigned char **out, size_t *outlen, X509 *x, + size_t chainidx, int *al, void *add_arg) +{ + ngx_connection_t *c; + ngx_quic_compat_t *com; + ngx_quic_connection_t *qc; + + c = ngx_ssl_get_connection(ssl); + if (c->type != SOCK_DGRAM) { + return 0; + } + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic compat add transport params"); + + qc = ngx_quic_get_connection(c); + com = qc->compat; + + *out = com->tp.data; + *outlen = com->tp.len; + + return 1; +} + + +static int +ngx_quic_compat_parse_transport_params_callback(SSL *ssl, unsigned int ext_type, + unsigned int context, const unsigned char *in, size_t inlen, X509 *x, + size_t chainidx, int *al, void *parse_arg) +{ + u_char *p; + ngx_connection_t *c; + ngx_quic_compat_t *com; + ngx_quic_connection_t *qc; + + c = ngx_ssl_get_connection(ssl); + if (c->type != SOCK_DGRAM) { + return 0; + } + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic compat parse transport params"); + + qc = ngx_quic_get_connection(c); + com = qc->compat; + + p = ngx_pnalloc(c->pool, inlen); + if (p == NULL) { + return 0; + } + + ngx_memcpy(p, in, inlen); + + com->ctp.data = p; + com->ctp.len = inlen; + + return 1; +} + + +int +SSL_set_quic_method(SSL *ssl, const SSL_QUIC_METHOD *quic_method) +{ + BIO *rbio, *wbio; + ngx_connection_t *c; + ngx_quic_compat_t *com; + ngx_quic_connection_t *qc; + + c = ngx_ssl_get_connection(ssl); + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic compat set method"); + + qc = ngx_quic_get_connection(c); + + qc->compat = ngx_pcalloc(c->pool, sizeof(ngx_quic_compat_t)); + if (qc->compat == NULL) { + return 0; + } + + com = qc->compat; + com->method = quic_method; + + rbio = BIO_new(BIO_s_mem()); + if (rbio == NULL) { + return 0; + } + + wbio = BIO_new(BIO_s_null()); + if (wbio == NULL) { + return 0; + } + + SSL_set_bio(ssl, rbio, wbio); + + SSL_set_msg_callback(ssl, ngx_quic_compat_message_callback); + + /* early data is not supported */ + SSL_set_max_early_data(ssl, 0); + + return 1; +} + + +static void +ngx_quic_compat_message_callback(int write_p, int version, int content_type, + const void *buf, size_t len, SSL *ssl, void *arg) +{ + ngx_uint_t alert; + ngx_connection_t *c; + ngx_quic_compat_t *com; + ngx_quic_connection_t *qc; + enum ssl_encryption_level_t level; + + if (!write_p) { + return; + } + + c = ngx_ssl_get_connection(ssl); + qc = ngx_quic_get_connection(c); + + if (qc == NULL) { + /* closing */ + return; + } + + com = qc->compat; + level = com->write_level; + + switch (content_type) { + + case SSL3_RT_HANDSHAKE: + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic compat tx %s len:%uz ", + ngx_quic_level_name(level), len); + + (void) com->method->add_handshake_data(ssl, level, buf, len); + + break; + + case SSL3_RT_ALERT: + if (len >= 2) { + alert = ((u_char *) buf)[1]; + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic compat %s alert:%ui len:%uz ", + ngx_quic_level_name(level), alert, len); + + (void) com->method->send_alert(ssl, level, alert); + + break; + } + + /* fall through */ + + default: + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic compat %s ignore msg:%d len:%uz ", + ngx_quic_level_name(level), content_type, len); + + break; + } +} + + +int +SSL_provide_quic_data(SSL *ssl, enum ssl_encryption_level_t level, + const uint8_t *data, size_t len) +{ + BIO *rbio; + size_t n; + u_char *p; + ngx_str_t res; + ngx_connection_t *c; + ngx_quic_compat_t *com; + ngx_quic_connection_t *qc; + ngx_quic_compat_record_t rec; + u_char in[NGX_QUIC_COMPAT_RECORD_SIZE + 1]; + u_char out[NGX_QUIC_COMPAT_RECORD_SIZE + 1 + + SSL3_RT_HEADER_LENGTH + + EVP_GCM_TLS_TAG_LEN]; + + c = ngx_ssl_get_connection(ssl); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic compat rx %s len:%uz", + ngx_quic_level_name(level), len); + + qc = ngx_quic_get_connection(c); + com = qc->compat; + rbio = SSL_get_rbio(ssl); + + while (len) { + ngx_memzero(&rec, sizeof(ngx_quic_compat_record_t)); + + rec.type = SSL3_RT_HANDSHAKE; + rec.log = c->log; + rec.number = com->read_record++; + rec.keys = &com->keys; + + if (level == ssl_encryption_initial) { + n = ngx_min(len, 65535); + + rec.payload.len = n; + rec.payload.data = (u_char *) data; + + ngx_quic_compat_create_header(&rec, out, 1); + + BIO_write(rbio, out, SSL3_RT_HEADER_LENGTH); + BIO_write(rbio, data, n); + +#if defined(NGX_QUIC_DEBUG_CRYPTO) && defined(NGX_QUIC_DEBUG_PACKETS) + ngx_log_debug5(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic compat record len:%uz %*xs%*xs", + n + SSL3_RT_HEADER_LENGTH, + (size_t) SSL3_RT_HEADER_LENGTH, out, n, data); +#endif + + } else { + n = ngx_min(len, NGX_QUIC_COMPAT_RECORD_SIZE); + + p = ngx_cpymem(in, data, n); + *p++ = SSL3_RT_HANDSHAKE; + + rec.payload.len = p - in; + rec.payload.data = in; + + res.data = out; + + if (ngx_quic_compat_create_record(&rec, &res) != NGX_OK) { + return 0; + } + +#if defined(NGX_QUIC_DEBUG_CRYPTO) && defined(NGX_QUIC_DEBUG_PACKETS) + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic compat record len:%uz %xV", res.len, &res); +#endif + + BIO_write(rbio, res.data, res.len); + } + + data += n; + len -= n; + } + + return 1; +} + + +static size_t +ngx_quic_compat_create_header(ngx_quic_compat_record_t *rec, u_char *out, + ngx_uint_t plain) +{ + u_char type; + size_t len; + + len = rec->payload.len; + + if (plain) { + type = rec->type; + + } else { + type = SSL3_RT_APPLICATION_DATA; + len += EVP_GCM_TLS_TAG_LEN; + } + + out[0] = type; + out[1] = 0x03; + out[2] = 0x03; + out[3] = (len >> 8); + out[4] = len; + + return 5; +} + + +static ngx_int_t +ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec, ngx_str_t *res) +{ + ngx_str_t ad, out; + ngx_quic_secret_t *secret; + ngx_quic_ciphers_t ciphers; + u_char nonce[NGX_QUIC_IV_LEN]; + + ad.data = res->data; + ad.len = ngx_quic_compat_create_header(rec, ad.data, 0); + + out.len = rec->payload.len + EVP_GCM_TLS_TAG_LEN; + out.data = res->data + ad.len; + +#ifdef NGX_QUIC_DEBUG_CRYPTO + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, rec->log, 0, + "quic compat ad len:%uz %xV", ad.len, &ad); +#endif + + if (ngx_quic_ciphers(rec->keys->cipher, &ciphers, rec->level) == NGX_ERROR) + { + return NGX_ERROR; + } + + secret = &rec->keys->secret; + + ngx_memcpy(nonce, secret->iv.data, secret->iv.len); + ngx_quic_compute_nonce(nonce, sizeof(nonce), rec->number); + + if (ngx_quic_tls_seal(ciphers.c, secret, &out, + nonce, &rec->payload, &ad, rec->log) + != NGX_OK) + { + return NGX_ERROR; + } + + res->len = ad.len + out.len; + + return NGX_OK; +} + + +enum ssl_encryption_level_t +SSL_quic_read_level(const SSL *ssl) +{ + ngx_connection_t *c; + ngx_quic_connection_t *qc; + + c = ngx_ssl_get_connection(ssl); + qc = ngx_quic_get_connection(c); + + return qc->compat->read_level; +} + + +enum ssl_encryption_level_t +SSL_quic_write_level(const SSL *ssl) +{ + ngx_connection_t *c; + ngx_quic_connection_t *qc; + + c = ngx_ssl_get_connection(ssl); + qc = ngx_quic_get_connection(c); + + return qc->compat->write_level; +} + + +int +SSL_set_quic_transport_params(SSL *ssl, const uint8_t *params, + size_t params_len) +{ + ngx_connection_t *c; + ngx_quic_compat_t *com; + ngx_quic_connection_t *qc; + + c = ngx_ssl_get_connection(ssl); + qc = ngx_quic_get_connection(c); + com = qc->compat; + + com->tp.len = params_len; + com->tp.data = (u_char *) params; + + return 1; +} + + +void +SSL_get_peer_quic_transport_params(const SSL *ssl, const uint8_t **out_params, + size_t *out_params_len) +{ + ngx_connection_t *c; + ngx_quic_compat_t *com; + ngx_quic_connection_t *qc; + + c = ngx_ssl_get_connection(ssl); + qc = ngx_quic_get_connection(c); + com = qc->compat; + + *out_params = com->ctp.data; + *out_params_len = com->ctp.len; +} + +#endif /* NGX_QUIC_OPENSSL_COMPAT */ diff --git a/src/event/quic/ngx_event_quic_openssl_compat.h b/src/event/quic/ngx_event_quic_openssl_compat.h new file mode 100644 --- /dev/null +++ b/src/event/quic/ngx_event_quic_openssl_compat.h @@ -0,0 +1,51 @@ + +/* + * Copyright (C) Nginx, Inc. + */ + + +#ifndef _NGX_EVENT_QUIC_OPENSSL_COMPAT_H_INCLUDED_ +#define _NGX_EVENT_QUIC_OPENSSL_COMPAT_H_INCLUDED_ + + +#include +#include + + +enum ssl_encryption_level_t { + ssl_encryption_initial = 0, + ssl_encryption_early_data, + ssl_encryption_handshake, + ssl_encryption_application +}; + + +typedef struct ssl_quic_method_st { + int (*set_read_secret)(SSL *ssl, enum ssl_encryption_level_t level, + const SSL_CIPHER *cipher, + const uint8_t *rsecret, size_t secret_len); + int (*set_write_secret)(SSL *ssl, enum ssl_encryption_level_t level, + const SSL_CIPHER *cipher, + const uint8_t *wsecret, size_t secret_len); + int (*add_handshake_data)(SSL *ssl, enum ssl_encryption_level_t level, + const uint8_t *data, size_t len); + int (*flush_flight)(SSL *ssl); + int (*send_alert)(SSL *ssl, enum ssl_encryption_level_t level, + uint8_t alert); +} SSL_QUIC_METHOD; + + +ngx_int_t ngx_quic_compat_init(SSL_CTX *ctx); + +int SSL_set_quic_method(SSL *ssl, const SSL_QUIC_METHOD *quic_method); +int SSL_provide_quic_data(SSL *ssl, enum ssl_encryption_level_t level, + const uint8_t *data, size_t len); +enum ssl_encryption_level_t SSL_quic_read_level(const SSL *ssl); +enum ssl_encryption_level_t SSL_quic_write_level(const SSL *ssl); +int SSL_set_quic_transport_params(SSL *ssl, const uint8_t *params, + size_t params_len); +void SSL_get_peer_quic_transport_params(const SSL *ssl, + const uint8_t **out_params, size_t *out_params_len); + + +#endif /* _NGX_EVENT_QUIC_OPENSSL_COMPAT_H_INCLUDED_ */ diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -23,37 +23,6 @@ #endif -#ifdef OPENSSL_IS_BORINGSSL -#define ngx_quic_cipher_t EVP_AEAD -#else -#define ngx_quic_cipher_t EVP_CIPHER -#endif - - -typedef struct { - const ngx_quic_cipher_t *c; - const EVP_CIPHER *hp; - const EVP_MD *d; -} ngx_quic_ciphers_t; - - -typedef struct { - size_t out_len; - u_char *out; - - size_t prk_len; - const uint8_t *prk; - - size_t label_len; - const u_char *label; -} ngx_quic_hkdf_t; - -#define ngx_quic_hkdf_set(seq, _label, _out, _prk) \ - (seq)->out_len = (_out)->len; (seq)->out = (_out)->data; \ - (seq)->prk_len = (_prk)->len, (seq)->prk = (_prk)->data, \ - (seq)->label_len = (sizeof(_label) - 1); (seq)->label = (u_char *)(_label); - - static ngx_int_t ngx_hkdf_expand(u_char *out_key, size_t out_len, const EVP_MD *digest, const u_char *prk, size_t prk_len, const u_char *info, size_t info_len); @@ -63,20 +32,12 @@ static ngx_int_t ngx_hkdf_extract(u_char static uint64_t ngx_quic_parse_pn(u_char **pos, ngx_int_t len, u_char *mask, uint64_t *largest_pn); -static void ngx_quic_compute_nonce(u_char *nonce, size_t len, uint64_t pn); -static ngx_int_t ngx_quic_ciphers(ngx_uint_t id, - ngx_quic_ciphers_t *ciphers, enum ssl_encryption_level_t level); static ngx_int_t ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); -static ngx_int_t ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, - ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, - ngx_str_t *ad, ngx_log_t *log); static ngx_int_t ngx_quic_tls_hp(ngx_log_t *log, const EVP_CIPHER *cipher, ngx_quic_secret_t *s, u_char *out, u_char *in); -static ngx_int_t ngx_quic_hkdf_expand(ngx_quic_hkdf_t *hkdf, - const EVP_MD *digest, ngx_log_t *log); static ngx_int_t ngx_quic_create_packet(ngx_quic_header_t *pkt, ngx_str_t *res); @@ -84,7 +45,7 @@ static ngx_int_t ngx_quic_create_retry_p ngx_str_t *res); -static ngx_int_t +ngx_int_t ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers, enum ssl_encryption_level_t level) { @@ -221,7 +182,7 @@ ngx_quic_keys_set_initial_secret(ngx_qui } -static ngx_int_t +ngx_int_t ngx_quic_hkdf_expand(ngx_quic_hkdf_t *h, const EVP_MD *digest, ngx_log_t *log) { size_t info_len; @@ -480,7 +441,7 @@ ngx_quic_tls_open(const ngx_quic_cipher_ } -static ngx_int_t +ngx_int_t ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) { @@ -961,7 +922,7 @@ ngx_quic_parse_pn(u_char **pos, ngx_int_ } -static void +void ngx_quic_compute_nonce(u_char *nonce, size_t len, uint64_t pn) { nonce[len - 8] ^= (pn >> 56) & 0x3f; diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -23,6 +23,13 @@ #define NGX_QUIC_MAX_MD_SIZE 48 +#ifdef OPENSSL_IS_BORINGSSL +#define ngx_quic_cipher_t EVP_AEAD +#else +#define ngx_quic_cipher_t EVP_CIPHER +#endif + + typedef struct { size_t len; u_char data[NGX_QUIC_MAX_MD_SIZE]; @@ -56,6 +63,30 @@ struct ngx_quic_keys_s { }; +typedef struct { + const ngx_quic_cipher_t *c; + const EVP_CIPHER *hp; + const EVP_MD *d; +} ngx_quic_ciphers_t; + + +typedef struct { + size_t out_len; + u_char *out; + + size_t prk_len; + const uint8_t *prk; + + size_t label_len; + const u_char *label; +} ngx_quic_hkdf_t; + +#define ngx_quic_hkdf_set(seq, _label, _out, _prk) \ + (seq)->out_len = (_out)->len; (seq)->out = (_out)->data; \ + (seq)->prk_len = (_prk)->len, (seq)->prk = (_prk)->data, \ + (seq)->label_len = (sizeof(_label) - 1); (seq)->label = (u_char *)(_label); + + ngx_int_t ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, ngx_log_t *log); ngx_int_t ngx_quic_keys_set_encryption_secret(ngx_log_t *log, @@ -70,6 +101,14 @@ void ngx_quic_keys_switch(ngx_connection ngx_int_t ngx_quic_keys_update(ngx_connection_t *c, ngx_quic_keys_t *keys); ngx_int_t ngx_quic_encrypt(ngx_quic_header_t *pkt, ngx_str_t *res); ngx_int_t ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn); +void ngx_quic_compute_nonce(u_char *nonce, size_t len, uint64_t pn); +ngx_int_t ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers, + enum ssl_encryption_level_t level); +ngx_int_t ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, + ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, + ngx_str_t *ad, ngx_log_t *log); +ngx_int_t ngx_quic_hkdf_expand(ngx_quic_hkdf_t *hkdf, const EVP_MD *digest, + ngx_log_t *log); #endif /* _NGX_EVENT_QUIC_PROTECTION_H_INCLUDED_ */ diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -10,6 +10,13 @@ #include +#if defined OPENSSL_IS_BORINGSSL \ + || defined LIBRESSL_VERSION_NUMBER \ + || NGX_QUIC_OPENSSL_COMPAT +#define NGX_QUIC_BORINGSSL_API 1 +#endif + + /* * RFC 9000, 7.5. Cryptographic Message Buffering * @@ -18,7 +25,7 @@ #define NGX_QUIC_MAX_BUFFERED 65535 -#if defined OPENSSL_IS_BORINGSSL || defined LIBRESSL_VERSION_NUMBER +#if (NGX_QUIC_BORINGSSL_API) static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len); @@ -39,7 +46,7 @@ static int ngx_quic_send_alert(ngx_ssl_c static ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data); -#if defined OPENSSL_IS_BORINGSSL || defined LIBRESSL_VERSION_NUMBER +#if (NGX_QUIC_BORINGSSL_API) static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, @@ -67,12 +74,6 @@ ngx_quic_set_read_secret(ngx_ssl_conn_t return 0; } - if (level == ssl_encryption_early_data) { - if (ngx_quic_init_streams(c) != NGX_OK) { - return 0; - } - } - return 1; } @@ -138,10 +139,6 @@ ngx_quic_set_encryption_secrets(ngx_ssl_ } if (level == ssl_encryption_early_data) { - if (ngx_quic_init_streams(c) != NGX_OK) { - return 0; - } - return 1; } @@ -456,6 +453,12 @@ ngx_quic_crypto_input(ngx_connection_t * return NGX_ERROR; } + if (ngx_quic_keys_available(qc->keys, ssl_encryption_early_data)) { + if (ngx_quic_init_streams(c) != NGX_OK) { + return NGX_ERROR; + } + } + return NGX_OK; } @@ -540,7 +543,7 @@ ngx_quic_init_connection(ngx_connection_ ssl_conn = c->ssl->connection; if (!quic_method.send_alert) { -#if defined OPENSSL_IS_BORINGSSL || defined LIBRESSL_VERSION_NUMBER +#if (NGX_QUIC_BORINGSSL_API) quic_method.set_read_secret = ngx_quic_set_read_secret; quic_method.set_write_secret = ngx_quic_set_write_secret; #else diff --git a/src/event/quic/ngx_event_quic_transport.h b/src/event/quic/ngx_event_quic_transport.h --- a/src/event/quic/ngx_event_quic_transport.h +++ b/src/event/quic/ngx_event_quic_transport.h @@ -11,6 +11,10 @@ #include #include +#if (NGX_QUIC_OPENSSL_COMPAT) +#include +#endif + /* * RFC 9000, 17.2. Long Header Packets From liebman at zod.com Mon Jan 9 17:44:52 2023 From: liebman at zod.com (Christopher Liebman) Date: Mon, 9 Jan 2023 09:44:52 -0800 Subject: [PATCH] limit_req_status: allow status response to be as low as 200 Message-ID: # HG changeset patch # User Christopher B. Liebman # Date 1672855906 28800 # Wed Jan 04 10:11:46 2023 -0800 # Node ID e438fab51202365305cace94aa25937081b051d0 # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 limit_req_status: allow status response to be as low as 200 No need to limit this to 400+ and it's handy to be able to return a non-error for for this. diff -r 07b0bee87f32 -r e438fab51202 src/http/modules/ngx_http_limit_req_module.c --- a/src/http/modules/ngx_http_limit_req_module.c Wed Dec 21 14:53:27 2022 +0300 +++ b/src/http/modules/ngx_http_limit_req_module.c Wed Jan 04 10:11:46 2023 -0800 @@ -97,7 +97,7 @@ static ngx_conf_num_bounds_t ngx_http_limit_req_status_bounds = { - ngx_conf_check_num_bounds, 400, 599 + ngx_conf_check_num_bounds, 200, 599 }; -------------- next part -------------- An HTML attachment was scrubbed... URL: From claireecf at gmail.com Mon Jan 9 17:59:50 2023 From: claireecf at gmail.com (claire liu) Date: Mon, 9 Jan 2023 11:59:50 -0600 Subject: congestion control algorithm Message-ID: Hi dev team, I have some questions about congestion control. What's the current congestion control algorithm in Nginx-quic? What are the differences between nginx and nginx-quic's congestion control algorithms? Thanks, Claire -------------- next part -------------- An HTML attachment was scrubbed... URL: From arut at nginx.com Tue Jan 10 13:28:02 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 10 Jan 2023 17:28:02 +0400 Subject: [PATCH 0 of 4] HTTP/3 insert count block improvements In-Reply-To: References: Message-ID: Updated patches. From arut at nginx.com Tue Jan 10 13:28:03 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 10 Jan 2023 17:28:03 +0400 Subject: [PATCH 1 of 4] QUIC: automatically add and never delete stream events In-Reply-To: References: Message-ID: # HG changeset patch # User Roman Arutyunyan # Date 1673345118 -14400 # Tue Jan 10 14:05:18 2023 +0400 # Branch quic # Node ID f89ce033b2fa161f7258f824dcf3c6ef2051f45f # Parent 987bee4363d10895f4bd1a40fc4347c49763e90f QUIC: automatically add and never delete stream events. Previously, stream events were added and deleted by ngx_handle_read_event() and ngx_handle_write_event() in a way similar to level-triggered events. However, QUIC stream events are effectively edge-triggered and can stay active all time. Moreover, the events are now active since the moment a stream is created. diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c --- a/src/event/ngx_event.c +++ b/src/event/ngx_event.c @@ -274,7 +274,7 @@ ngx_handle_read_event(ngx_event_t *rev, c = rev->data; if (c->quic) { - return ngx_quic_handle_read_event(rev, flags); + return NGX_OK; } #endif @@ -353,7 +353,7 @@ ngx_handle_write_event(ngx_event_t *wev, #if (NGX_QUIC) if (c->quic) { - return ngx_quic_handle_write_event(wev, lowat); + return NGX_OK; } #endif diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h --- a/src/event/quic/ngx_event_quic.h +++ b/src/event/quic/ngx_event_quic.h @@ -123,8 +123,6 @@ void ngx_quic_shutdown_connection(ngx_co ngx_int_t ngx_quic_reset_stream(ngx_connection_t *c, ngx_uint_t err); ngx_int_t ngx_quic_shutdown_stream(ngx_connection_t *c, int how); void ngx_quic_cancelable_stream(ngx_connection_t *c); -ngx_int_t ngx_quic_handle_read_event(ngx_event_t *rev, ngx_uint_t flags); -ngx_int_t ngx_quic_handle_write_event(ngx_event_t *wev, size_t lowat); ngx_int_t ngx_quic_get_packet_dcid(ngx_log_t *log, u_char *data, size_t len, ngx_str_t *dcid); ngx_int_t ngx_quic_derive_key(ngx_log_t *log, const char *label, diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c --- a/src/event/quic/ngx_event_quic_streams.c +++ b/src/event/quic/ngx_event_quic_streams.c @@ -699,9 +699,16 @@ ngx_quic_create_stream(ngx_connection_t if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0 || (id & NGX_QUIC_STREAM_SERVER_INITIATED)) { + sc->write->active = 1; sc->write->ready = 1; } + if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0 + || (id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0) + { + sc->read->active = 1; + } + if (id & NGX_QUIC_STREAM_UNIDIRECTIONAL) { if (id & NGX_QUIC_STREAM_SERVER_INITIATED) { qs->send_max_data = qc->ctp.initial_max_stream_data_uni; @@ -1745,31 +1752,3 @@ ngx_quic_set_event(ngx_event_t *ev) ngx_post_event(ev, &ngx_posted_events); } } - - -ngx_int_t -ngx_quic_handle_read_event(ngx_event_t *rev, ngx_uint_t flags) -{ - if (!rev->active && !rev->ready) { - rev->active = 1; - - } else if (rev->active && (rev->ready || (flags & NGX_CLOSE_EVENT))) { - rev->active = 0; - } - - return NGX_OK; -} - - -ngx_int_t -ngx_quic_handle_write_event(ngx_event_t *wev, size_t lowat) -{ - if (!wev->active && !wev->ready) { - wev->active = 1; - - } else if (wev->active && wev->ready) { - wev->active = 0; - } - - return NGX_OK; -} From arut at nginx.com Tue Jan 10 13:28:04 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 10 Jan 2023 17:28:04 +0400 Subject: [PATCH 2 of 4] QUIC: set stream error flag on reset In-Reply-To: References: Message-ID: <18cc0fb5509d09d241d1.1673357284@arut-laptop> # HG changeset patch # User Roman Arutyunyan # Date 1672932528 -14400 # Thu Jan 05 19:28:48 2023 +0400 # Branch quic # Node ID 18cc0fb5509d09d241d1235c754031304fa51685 # Parent f89ce033b2fa161f7258f824dcf3c6ef2051f45f QUIC: set stream error flag on reset. Now, when RESET_STREAM is sent or received, or when streams are closed, stream connection error flag is set. Previously, only stream state was changed, which resulted in setting the error flag only after calling recv()/send()/send_chain(). However, there are cases when none of these functions is called, but it's still important to know if the stream is being closed. For example, when an HTTP/3 request stream is blocked on insert count, receiving RESET_STREAM should trigger stream closure, which was not the case. The change also fixes ngx_http_test_reading() and ngx_http_upstream_check_broken_connection() with QUIC streams. diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c --- a/src/event/quic/ngx_event_quic_streams.c +++ b/src/event/quic/ngx_event_quic_streams.c @@ -203,6 +203,9 @@ ngx_quic_close_streams(ngx_connection_t continue; } + sc->read->error = 1; + sc->write->error = 1; + ngx_quic_set_event(sc->read); ngx_quic_set_event(sc->write); @@ -245,6 +248,10 @@ ngx_quic_do_reset_stream(ngx_quic_stream qs->send_state = NGX_QUIC_STREAM_SEND_RESET_SENT; qs->send_final_size = qs->send_offset; + if (qs->connection) { + qs->connection->write->error = 1; + } + pc = qs->parent; qc = ngx_quic_get_connection(pc); @@ -804,7 +811,6 @@ ngx_quic_stream_recv(ngx_connection_t *c || qs->recv_state == NGX_QUIC_STREAM_RECV_RESET_READ) { qs->recv_state = NGX_QUIC_STREAM_RECV_RESET_READ; - rev->error = 1; return NGX_ERROR; } @@ -1382,6 +1388,7 @@ ngx_int_t ngx_quic_handle_reset_stream_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_reset_stream_frame_t *f) { + ngx_event_t *rev; ngx_quic_stream_t *qs; ngx_quic_connection_t *qc; @@ -1438,7 +1445,10 @@ ngx_quic_handle_reset_stream_frame(ngx_c return ngx_quic_close_stream(qs); } - ngx_quic_set_event(qs->connection->read); + rev = qs->connection->read; + rev->error = 1; + + ngx_quic_set_event(rev); return NGX_OK; } From arut at nginx.com Tue Jan 10 13:28:05 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 10 Jan 2023 17:28:05 +0400 Subject: [PATCH 3 of 4] HTTP/3: trigger 400 (Bad Request) on stream error while blocked In-Reply-To: References: Message-ID: <7358ff656a6b12416692.1673357285@arut-laptop> # HG changeset patch # User Roman Arutyunyan # Date 1672928146 -14400 # Thu Jan 05 18:15:46 2023 +0400 # Branch quic # Node ID 7358ff656a6b124166926c026a13ec3057b6db6b # Parent 18cc0fb5509d09d241d1235c754031304fa51685 HTTP/3: trigger 400 (Bad Request) on stream error while blocked. Previously, stream was closed with NGX_HTTP_CLOSE. However, in a similar case when recv() returns eof or error, status 400 is triggered. diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -554,7 +554,7 @@ ngx_http_v3_process_request(ngx_event_t if (rc == NGX_BUSY) { if (rev->error) { - ngx_http_close_request(r, NGX_HTTP_CLOSE); + ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); break; } From arut at nginx.com Tue Jan 10 13:28:06 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 10 Jan 2023 17:28:06 +0400 Subject: [PATCH 4 of 4] HTTP/3: insert count block timeout In-Reply-To: References: Message-ID: <7dac46ecb27d3f488d40.1673357286@arut-laptop> # HG changeset patch # User Roman Arutyunyan # Date 1672931002 -14400 # Thu Jan 05 19:03:22 2023 +0400 # Branch quic # Node ID 7dac46ecb27d3f488d40b28cd8d07f884046e171 # Parent 7358ff656a6b124166926c026a13ec3057b6db6b HTTP/3: insert count block timeout. Previously, there was no timeout for a request stream blocked on insert count, which could result in infinite wait. Now client_header_timeout is set when stream is first blocked. diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -558,6 +558,12 @@ ngx_http_v3_process_request(ngx_event_t break; } + if (!rev->timer_set) { + cscf = ngx_http_get_module_srv_conf(r, + ngx_http_core_module); + ngx_add_timer(rev, cscf->client_header_timeout); + } + if (ngx_handle_read_event(rev, 0) != NGX_OK) { ngx_http_close_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); } From arut at nginx.com Tue Jan 10 13:30:54 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 10 Jan 2023 17:30:54 +0400 Subject: [PATCH] QUIC: relocated ngx_quic_init_streams() for 0-RTT Message-ID: <5a09008e93a3842dc5a3.1673357454@arut-laptop> # HG changeset patch # User Roman Arutyunyan # Date 1673357050 -14400 # Tue Jan 10 17:24:10 2023 +0400 # Branch quic # Node ID 5a09008e93a3842dc5a372e39617097893293ddf # Parent 7dac46ecb27d3f488d40b28cd8d07f884046e171 QUIC: relocated ngx_quic_init_streams() for 0-RTT. Previously, streams were initialized in early keys handler. However, client transport parameters may not be available by then. This happens, for example, when using QuicTLS. Now streams are initialized in ngx_quic_crypto_input() after calling SSL_do_handshake() for both 0-RTT and 1-RTT. diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -67,12 +67,6 @@ ngx_quic_set_read_secret(ngx_ssl_conn_t return 0; } - if (level == ssl_encryption_early_data) { - if (ngx_quic_init_streams(c) != NGX_OK) { - return 0; - } - } - return 1; } @@ -138,10 +132,6 @@ ngx_quic_set_encryption_secrets(ngx_ssl_ } if (level == ssl_encryption_early_data) { - if (ngx_quic_init_streams(c) != NGX_OK) { - return 0; - } - return 1; } @@ -455,11 +445,17 @@ ngx_quic_crypto_input(ngx_connection_t * qc->error_reason = "handshake failed"; return NGX_ERROR; } - - return NGX_OK; } - if (SSL_in_init(ssl_conn)) { + if (n <= 0 || SSL_in_init(ssl_conn)) { + if (ngx_quic_keys_available(qc->keys, ssl_encryption_early_data) + && qc->client_tp_done) + { + if (ngx_quic_init_streams(c) != NGX_OK) { + return NGX_ERROR; + } + } + return NGX_OK; } From pluknet at nginx.com Tue Jan 10 13:34:03 2023 From: pluknet at nginx.com (=?iso-8859-1?q?Sergey_Kandaurov?=) Date: Tue, 10 Jan 2023 17:34:03 +0400 Subject: [PATCH] QUIC: fixed $connection_time Message-ID: <267dbbc62495f5481224.1673357643@enoparse.local> # HG changeset patch # User Sergey Kandaurov # Date 1673354510 -14400 # Tue Jan 10 16:41:50 2023 +0400 # Branch quic # Node ID 267dbbc62495f5481224c956145697a91364e532 # Parent 19633ad7b73cba34e61f5619193cbcac2f949947 QUIC: fixed $connection_time. Previously, start_time wasn't set for a new stream. The fix is to derive it from the parent connection. Additionally, it's used to simplify keepalive_time. diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c --- a/src/event/quic/ngx_event_quic_streams.c +++ b/src/event/quic/ngx_event_quic_streams.c @@ -689,6 +689,7 @@ ngx_quic_create_stream(ngx_connection_t sc->local_sockaddr = c->local_sockaddr; sc->local_socklen = c->local_socklen; sc->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); + sc->start_time = c->start_time; sc->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; sc->recv = ngx_quic_stream_recv; diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -165,7 +165,6 @@ ngx_http_v3_init_request_stream(ngx_conn { uint64_t n; ngx_event_t *rev; - ngx_connection_t *pc; ngx_pool_cleanup_t *cln; ngx_http_connection_t *hc; ngx_http_v3_session_t *h3c; @@ -199,12 +198,10 @@ ngx_http_v3_init_request_stream(ngx_conn return; } - pc = c->quic->parent; - h3c->next_request_id = c->quic->id + 0x04; if (n + 1 == clcf->keepalive_requests - || ngx_current_msec - pc->start_time > clcf->keepalive_time) + || ngx_current_msec - c->start_time > clcf->keepalive_time) { h3c->goaway = 1; From pluknet at nginx.com Tue Jan 10 13:38:25 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Tue, 10 Jan 2023 17:38:25 +0400 Subject: [PATCH 0 of 4] HTTP/3 insert count block improvements In-Reply-To: References: Message-ID: <671C6E10-5794-47C7-9C2F-A04291BAB6EB@nginx.com> > On 10 Jan 2023, at 17:28, Roman Arutyunyan wrote: > > Updated patches. Looks good. -- Sergey Kandaurov From arut at nginx.com Tue Jan 10 13:40:55 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Tue, 10 Jan 2023 17:40:55 +0400 Subject: [PATCH] QUIC: fixed $connection_time In-Reply-To: <267dbbc62495f5481224.1673357643@enoparse.local> References: <267dbbc62495f5481224.1673357643@enoparse.local> Message-ID: <20230110134055.h37di4vxqhexe2z4@N00W24XTQX> On Tue, Jan 10, 2023 at 05:34:03PM +0400, Sergey Kandaurov wrote: > # HG changeset patch > # User Sergey Kandaurov > # Date 1673354510 -14400 > # Tue Jan 10 16:41:50 2023 +0400 > # Branch quic > # Node ID 267dbbc62495f5481224c956145697a91364e532 > # Parent 19633ad7b73cba34e61f5619193cbcac2f949947 > QUIC: fixed $connection_time. > > Previously, start_time wasn't set for a new stream. > The fix is to derive it from the parent connection. > Additionally, it's used to simplify keepalive_time. tracking keepalive_time? > diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c > --- a/src/event/quic/ngx_event_quic_streams.c > +++ b/src/event/quic/ngx_event_quic_streams.c > @@ -689,6 +689,7 @@ ngx_quic_create_stream(ngx_connection_t > sc->local_sockaddr = c->local_sockaddr; > sc->local_socklen = c->local_socklen; > sc->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); > + sc->start_time = c->start_time; > sc->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; > > sc->recv = ngx_quic_stream_recv; > diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c > --- a/src/http/v3/ngx_http_v3_request.c > +++ b/src/http/v3/ngx_http_v3_request.c > @@ -165,7 +165,6 @@ ngx_http_v3_init_request_stream(ngx_conn > { > uint64_t n; > ngx_event_t *rev; > - ngx_connection_t *pc; > ngx_pool_cleanup_t *cln; > ngx_http_connection_t *hc; > ngx_http_v3_session_t *h3c; > @@ -199,12 +198,10 @@ ngx_http_v3_init_request_stream(ngx_conn > return; > } > > - pc = c->quic->parent; > - > h3c->next_request_id = c->quic->id + 0x04; > > if (n + 1 == clcf->keepalive_requests > - || ngx_current_msec - pc->start_time > clcf->keepalive_time) > + || ngx_current_msec - c->start_time > clcf->keepalive_time) > { > h3c->goaway = 1; > > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel Looks ok -- Roman Arutyunyan From arut at nginx.com Wed Jan 11 08:55:30 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Wed, 11 Jan 2023 12:55:30 +0400 Subject: congestion control algorithm In-Reply-To: References: Message-ID: <0023372A-EB41-428F-B17A-D3709FF3059E@nginx.com> Hi Claire, > On 9 Jan 2023, at 21:59, claire liu wrote: > > Hi dev team, > I have some questions about congestion control. What's the current congestion control algorithm in Nginx-quic? What are the differences between nginx and nginx-quic's congestion control algorithms? nginx-quic implements NewReno congestion control as per RFC 9002, Section 7. We have plans to add more advanced algorithms though. Nginx does not have any congestion control implemented, since it relies on the kernel to implement one for TCP. > > Thanks, > Claire > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel ---- Roman Arutyunyan arut at nginx.com -------------- next part -------------- An HTML attachment was scrubbed... URL: From mdounin at mdounin.ru Wed Jan 11 20:22:01 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Wed, 11 Jan 2023 23:22:01 +0300 Subject: [PATCH] limit_req_status: allow status response to be as low as 200 In-Reply-To: References: Message-ID: Hello! On Mon, Jan 09, 2023 at 09:44:52AM -0800, Christopher Liebman wrote: > # HG changeset patch > # User Christopher B. Liebman > # Date 1672855906 28800 > # Wed Jan 04 10:11:46 2023 -0800 > # Node ID e438fab51202365305cace94aa25937081b051d0 > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > limit_req_status: allow status response to be as low as 200 > > No need to limit this to 400+ and it's handy to be able to return > a non-error for for this. > > diff -r 07b0bee87f32 -r e438fab51202 > src/http/modules/ngx_http_limit_req_module.c > --- a/src/http/modules/ngx_http_limit_req_module.c Wed Dec 21 14:53:27 2022 > +0300 > +++ b/src/http/modules/ngx_http_limit_req_module.c Wed Jan 04 10:11:46 2023 > -0800 > @@ -97,7 +97,7 @@ > > > static ngx_conf_num_bounds_t ngx_http_limit_req_status_bounds = { > - ngx_conf_check_num_bounds, 400, 599 > + ngx_conf_check_num_bounds, 200, 599 > }; Thank you for the patch. Unfortunately, removing limitation is not going to work, since returning non-error status codes also requires sending an actual response. The good part is that you can easily return any responses by using an error status and configuring appropriate error_page, see http://nginx.org/r/error_page for details. -- Maxim Dounin http://mdounin.ru/ From liebman at zod.com Thu Jan 12 13:16:21 2023 From: liebman at zod.com (Christopher Liebman) Date: Thu, 12 Jan 2023 05:16:21 -0800 Subject: [PATCH] limit_req_status: allow status response to be as low as 200 In-Reply-To: References: Message-ID: Not with 204. This works quite well with a partner that has an aversion to errors when they run over the limit: limit_req_status 204; On Wed, Jan 11, 2023 at 12:22 PM Maxim Dounin wrote: > Hello! > > On Mon, Jan 09, 2023 at 09:44:52AM -0800, Christopher Liebman wrote: > > > # HG changeset patch > > # User Christopher B. Liebman > > # Date 1672855906 28800 > > # Wed Jan 04 10:11:46 2023 -0800 > > # Node ID e438fab51202365305cace94aa25937081b051d0 > > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > > limit_req_status: allow status response to be as low as 200 > > > > No need to limit this to 400+ and it's handy to be able to return > > a non-error for for this. > > > > diff -r 07b0bee87f32 -r e438fab51202 > > src/http/modules/ngx_http_limit_req_module.c > > --- a/src/http/modules/ngx_http_limit_req_module.c Wed Dec 21 14:53:27 > 2022 > > +0300 > > +++ b/src/http/modules/ngx_http_limit_req_module.c Wed Jan 04 10:11:46 > 2023 > > -0800 > > @@ -97,7 +97,7 @@ > > > > > > static ngx_conf_num_bounds_t ngx_http_limit_req_status_bounds = { > > - ngx_conf_check_num_bounds, 400, 599 > > + ngx_conf_check_num_bounds, 200, 599 > > }; > > Thank you for the patch. > > Unfortunately, removing limitation is not going to work, since > returning non-error status codes also requires sending an actual > response. > > The good part is that you can easily return any responses by using > an error status and configuring appropriate error_page, see > http://nginx.org/r/error_page for details. > > -- > Maxim Dounin > http://mdounin.ru/ > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel > -------------- next part -------------- An HTML attachment was scrubbed... URL: From mdounin at mdounin.ru Thu Jan 12 17:11:49 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 12 Jan 2023 20:11:49 +0300 Subject: [PATCH] limit_req_status: allow status response to be as low as 200 In-Reply-To: References: Message-ID: Hello! On Thu, Jan 12, 2023 at 05:16:21AM -0800, Christopher Liebman wrote: > Not with 204. > This works quite well with a partner that has an aversion to errors when > they run over the limit: > limit_req_status 204; Indeed, 204 happens to be one of the two 2xx codes which can be returned directly, as they are handled in ngx_http_finalize_request() to facilitate simpler code in the dav module. This is not what your patch enables though. For all other codes, except 204 and 201 mentioned above, just returning them will simply break things. As already suggested, proper behaviour for all the codes can be easily configured by using the "error_page" directive. Hope this helps. -- Maxim Dounin http://mdounin.ru/ From jordanc.carter at outlook.com Thu Jan 12 21:35:59 2023 From: jordanc.carter at outlook.com (J Carter) Date: Thu, 12 Jan 2023 21:35:59 +0000 Subject: [PATCH] limit_req_status: allow status response to be as low as 200 In-Reply-To: References: Message-ID: It's trivial to do with error page OP as mentioned - an example:     limit_req_status 598;     limit_req_zone $binary_remote_addr zone=a:1m rate=1r/m;     server {         limit_req zone=a nodelay;         error_page 598 = @send-204;         location / {                ...         }         ...         location @send-204 {                 return 204;         }     } error_page's '=' can also be set to 204, and the named location contain nothing at all (whatever you prefer). On 12/01/2023 17:11, Maxim Dounin wrote: > Hello! > > On Thu, Jan 12, 2023 at 05:16:21AM -0800, Christopher Liebman wrote: > >> Not with 204. >> This works quite well with a partner that has an aversion to errors when >> they run over the limit: >> limit_req_status 204; > Indeed, 204 happens to be one of the two 2xx codes which can be > returned directly, as they are handled in > ngx_http_finalize_request() to facilitate simpler > code in the dav module. This is not what your patch enables > though. For all other codes, except 204 and 201 mentioned above, > just returning them will simply break things. > > As already suggested, proper behaviour for all the codes can be > easily configured by using the "error_page" directive. > > Hope this helps. > From mdounin at mdounin.ru Thu Jan 12 21:35:23 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:23 +0300 Subject: [PATCH 00 of 12] win32 non-ASCII names support fixes Message-ID: Hello! The following patch series fixes various issues with non-ASCII names on Windows. Notably, the following Trac tickets are addressed: "Win32: autoindex module doesn't support Unicode names" https://trac.nginx.org/nginx/ticket/458 "WebDAV module didn't convert UTF8 encode url into GBK on Windows" https://trac.nginx.org/nginx/ticket/1433 Further, it resolves various issues with using directories with non-ASCII names in nginx configuration, such as in prefix, directories for temporary files, and include files with wildcards. -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Thu Jan 12 21:35:26 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:26 +0300 Subject: [PATCH 03 of 12] Win32: non-ASCII directory names support in ngx_getcwd() In-Reply-To: References: Message-ID: <7cf820c46860796cff91.1673559326@1.0.0.127.in-addr.arpa> # HG changeset patch # User Maxim Dounin # Date 1673548916 -10800 # Thu Jan 12 21:41:56 2023 +0300 # Node ID 7cf820c46860796cff91f53a5d2db669bb5b5a6c # Parent d05c0adf5890aecc68ce8906ef19ca07502ed071 Win32: non-ASCII directory names support in ngx_getcwd(). This makes it possible to start nginx without a prefix explicitly set in a directory with non-ASCII characters in it. diff -r d05c0adf5890 -r 7cf820c46860 src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:41:39 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:41:56 2023 +0300 @@ -428,6 +428,30 @@ ngx_realpath(u_char *path, u_char *resol } +size_t +ngx_getcwd(u_char *buf, size_t size) +{ + u_char *p; + size_t n; + u_short utf16[NGX_MAX_PATH]; + + n = GetCurrentDirectoryW(NGX_MAX_PATH, utf16); + + if (n == 0) { + return 0; + } + + p = ngx_utf16_to_utf8(buf, utf16, &size, NULL); + + if (p != buf) { + ngx_free(p); + return 0; + } + + return size - 1; +} + + ngx_int_t ngx_open_dir(ngx_str_t *name, ngx_dir_t *dir) { diff -r d05c0adf5890 -r 7cf820c46860 src/os/win32/ngx_files.h --- a/src/os/win32/ngx_files.h Thu Jan 12 21:41:39 2023 +0300 +++ b/src/os/win32/ngx_files.h Thu Jan 12 21:41:56 2023 +0300 @@ -178,8 +178,12 @@ void ngx_close_file_mapping(ngx_file_map u_char *ngx_realpath(u_char *path, u_char *resolved); #define ngx_realpath_n "" -#define ngx_getcwd(buf, size) GetCurrentDirectory(size, (char *) buf) + + +size_t ngx_getcwd(u_char *buf, size_t size); #define ngx_getcwd_n "GetCurrentDirectory()" + + #define ngx_path_separator(c) ((c) == '/' || (c) == '\\') #define NGX_HAVE_MAX_PATH 1 From mdounin at mdounin.ru Thu Jan 12 21:35:24 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:24 +0300 Subject: [PATCH 01 of 12] Win32: non-ASCII names support in autoindex (ticket #458) In-Reply-To: References: Message-ID: <60d845f9505fe1b97c1e.1673559324@1.0.0.127.in-addr.arpa> # HG changeset patch # User Maxim Dounin # Date 1673548890 -10800 # Thu Jan 12 21:41:30 2023 +0300 # Node ID 60d845f9505fe1b97c1e04b680523b790e29fdb1 # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 Win32: non-ASCII names support in autoindex (ticket #458). Notably, ngx_open_dir() now supports opening directories with non-ASCII characters, and directory entries returned by ngx_read_dir() are properly converted to UTF-8. diff -r 07b0bee87f32 -r 60d845f9505f src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Wed Dec 21 14:53:27 2022 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:41:30 2023 +0300 @@ -13,7 +13,11 @@ static ngx_int_t ngx_win32_check_filename(u_char *name, u_short *u, size_t len); -static u_short *ngx_utf8_to_utf16(u_short *utf16, u_char *utf8, size_t *len); +static u_short *ngx_utf8_to_utf16(u_short *utf16, u_char *utf8, size_t *len, + size_t reserved); +static u_char *ngx_utf16_to_utf8(u_char *utf8, u_short *utf16, size_t *len, + size_t *allocated); +uint32_t ngx_utf16_decode(u_short **u, size_t n); /* FILE_FLAG_BACKUP_SEMANTICS allows to obtain a handle to a directory */ @@ -28,7 +32,7 @@ ngx_open_file(u_char *name, u_long mode, u_short utf16[NGX_UTF16_BUFLEN]; len = NGX_UTF16_BUFLEN; - u = ngx_utf8_to_utf16(utf16, name, &len); + u = ngx_utf8_to_utf16(utf16, name, &len, 0); if (u == NULL) { return INVALID_HANDLE_VALUE; @@ -269,7 +273,7 @@ ngx_file_info(u_char *file, ngx_file_inf len = NGX_UTF16_BUFLEN; - u = ngx_utf8_to_utf16(utf16, file, &len); + u = ngx_utf8_to_utf16(utf16, file, &len, 0); if (u == NULL) { return NGX_FILE_ERROR; @@ -427,49 +431,51 @@ ngx_realpath(u_char *path, u_char *resol ngx_int_t ngx_open_dir(ngx_str_t *name, ngx_dir_t *dir) { - u_char *pattern, *p; + size_t len; + u_short *u, *p; ngx_err_t err; + u_short utf16[NGX_UTF16_BUFLEN]; - pattern = malloc(name->len + 3); - if (pattern == NULL) { + len = NGX_UTF16_BUFLEN - 2; + u = ngx_utf8_to_utf16(utf16, name->data, &len, 2); + + if (u == NULL) { return NGX_ERROR; } - p = ngx_cpymem(pattern, name->data, name->len); + if (ngx_win32_check_filename(name->data, u, len) != NGX_OK) { + goto failed; + } + + p = &u[len - 1]; *p++ = '/'; *p++ = '*'; *p = '\0'; - dir->dir = FindFirstFile((const char *) pattern, &dir->finddata); + dir->dir = FindFirstFileW(u, &dir->finddata); if (dir->dir == INVALID_HANDLE_VALUE) { - err = ngx_errno; - ngx_free(pattern); - ngx_set_errno(err); - return NGX_ERROR; + goto failed; } - ngx_free(pattern); + if (u != utf16) { + ngx_free(u); + } dir->valid_info = 1; dir->ready = 1; + dir->name = NULL; + dir->allocated = 0; return NGX_OK; -} +failed: -ngx_int_t -ngx_read_dir(ngx_dir_t *dir) -{ - if (dir->ready) { - dir->ready = 0; - return NGX_OK; - } - - if (FindNextFile(dir->dir, &dir->finddata) != 0) { - dir->type = 1; - return NGX_OK; + if (u != utf16) { + err = ngx_errno; + ngx_free(u); + ngx_set_errno(err); } return NGX_ERROR; @@ -477,8 +483,56 @@ ngx_read_dir(ngx_dir_t *dir) ngx_int_t +ngx_read_dir(ngx_dir_t *dir) +{ + u_char *name; + size_t len, allocated; + + if (dir->ready) { + dir->ready = 0; + goto convert; + } + + if (FindNextFileW(dir->dir, &dir->finddata) != 0) { + dir->type = 1; + goto convert; + } + + return NGX_ERROR; + +convert: + + name = dir->name; + len = dir->allocated; + + name = ngx_utf16_to_utf8(name, dir->finddata.cFileName, &len, &allocated); + if (name == NULL) { + return NGX_ERROR; + } + + if (name != dir->name) { + + if (dir->name) { + ngx_free(dir->name); + } + + dir->name = name; + dir->allocated = allocated; + } + + dir->namelen = len - 1; + + return NGX_OK; +} + + +ngx_int_t ngx_close_dir(ngx_dir_t *dir) { + if (dir->name) { + ngx_free(dir->name); + } + if (FindClose(dir->dir) == 0) { return NGX_ERROR; } @@ -816,7 +870,7 @@ failed: static u_short * -ngx_utf8_to_utf16(u_short *utf16, u_char *utf8, size_t *len) +ngx_utf8_to_utf16(u_short *utf16, u_char *utf8, size_t *len, size_t reserved) { u_char *p; u_short *u, *last; @@ -865,7 +919,7 @@ ngx_utf8_to_utf16(u_short *utf16, u_char /* the given buffer is not enough, allocate a new one */ - u = malloc(((p - utf8) + ngx_strlen(p) + 1) * sizeof(u_short)); + u = malloc(((p - utf8) + ngx_strlen(p) + 1 + reserved) * sizeof(u_short)); if (u == NULL) { return NULL; } @@ -910,3 +964,170 @@ ngx_utf8_to_utf16(u_short *utf16, u_char /* unreachable */ } + + +static u_char * +ngx_utf16_to_utf8(u_char *utf8, u_short *utf16, size_t *len, size_t *allocated) +{ + u_char *p, *last; + u_short *u, *j; + uint32_t n; + + u = utf16; + p = utf8; + last = utf8 + *len; + + while (p < last) { + + if (*u < 0x80) { + *p++ = (u_char) *u; + + if (*u == 0) { + *len = p - utf8; + return utf8; + } + + u++; + + continue; + } + + if (p >= last - 4) { + *len = p - utf8; + break; + } + + n = ngx_utf16_decode(&u, 2); + + if (n > 0x10ffff) { + ngx_free(utf8); + ngx_set_errno(NGX_EILSEQ); + return NULL; + } + + if (n >= 0x10000) { + *p++ = (u_char) (0xf0 + (n >> 18)); + *p++ = (u_char) (0x80 + ((n >> 12) & 0x3f)); + *p++ = (u_char) (0x80 + ((n >> 6) & 0x3f)); + *p++ = (u_char) (0x80 + (n & 0x3f)); + continue; + + } + + if (n >= 0x0800) { + *p++ = (u_char) (0xe0 + (n >> 12)); + *p++ = (u_char) (0x80 + ((n >> 6) & 0x3f)); + *p++ = (u_char) (0x80 + (n & 0x3f)); + continue; + } + + *p++ = (u_char) (0xc0 + (n >> 6)); + *p++ = (u_char) (0x80 + (n & 0x3f)); + } + + /* the given buffer is not enough, allocate a new one */ + + for (j = u; *j; j++) { /* void */ } + + p = malloc((j - utf16) * 4 + 1); + if (p == NULL) { + return NULL; + } + + if (allocated) { + *allocated = (j - utf16) * 4 + 1; + } + + ngx_memcpy(p, utf8, *len); + + utf8 = p; + p += *len; + + for ( ;; ) { + + if (*u < 0x80) { + *p++ = (u_char) *u; + + if (*u == 0) { + *len = p - utf8; + return utf8; + } + + u++; + + continue; + } + + n = ngx_utf16_decode(&u, 2); + + if (n > 0x10ffff) { + ngx_free(utf8); + ngx_set_errno(NGX_EILSEQ); + return NULL; + } + + if (n >= 0x10000) { + *p++ = (u_char) (0xf0 + (n >> 18)); + *p++ = (u_char) (0x80 + ((n >> 12) & 0x3f)); + *p++ = (u_char) (0x80 + ((n >> 6) & 0x3f)); + *p++ = (u_char) (0x80 + (n & 0x3f)); + continue; + + } + + if (n >= 0x0800) { + *p++ = (u_char) (0xe0 + (n >> 12)); + *p++ = (u_char) (0x80 + ((n >> 6) & 0x3f)); + *p++ = (u_char) (0x80 + (n & 0x3f)); + continue; + } + + *p++ = (u_char) (0xc0 + (n >> 6)); + *p++ = (u_char) (0x80 + (n & 0x3f)); + } + + /* unreachable */ +} + + +/* + * ngx_utf16_decode() decodes one or two UTF-16 code units + * the return values: + * 0x80 - 0x10ffff valid character + * 0x110000 - 0xfffffffd invalid sequence + * 0xfffffffe incomplete sequence + * 0xffffffff error + */ + +uint32_t +ngx_utf16_decode(u_short **u, size_t n) +{ + uint32_t k, m; + + k = **u; + + if (k < 0xd800 || k > 0xdfff) { + (*u)++; + return k; + } + + if (k > 0xdbff) { + (*u)++; + return 0xffffffff; + } + + if (n < 2) { + return 0xfffffffe; + } + + (*u)++; + + m = *(*u)++; + + if (m < 0xdc00 || m > 0xdfff) { + return 0xffffffff; + + } + + return 0x10000 + ((k - 0xd800) << 10) + (m - 0xdc00); +} diff -r 07b0bee87f32 -r 60d845f9505f src/os/win32/ngx_files.h --- a/src/os/win32/ngx_files.h Wed Dec 21 14:53:27 2022 +0300 +++ b/src/os/win32/ngx_files.h Thu Jan 12 21:41:30 2023 +0300 @@ -30,7 +30,11 @@ typedef struct { typedef struct { HANDLE dir; - WIN32_FIND_DATA finddata; + WIN32_FIND_DATAW finddata; + + u_char *name; + size_t namelen; + size_t allocated; unsigned valid_info:1; unsigned type:1; @@ -205,8 +209,8 @@ ngx_int_t ngx_close_dir(ngx_dir_t *dir); #define ngx_dir_access(a) (a) -#define ngx_de_name(dir) ((u_char *) (dir)->finddata.cFileName) -#define ngx_de_namelen(dir) ngx_strlen((dir)->finddata.cFileName) +#define ngx_de_name(dir) (dir)->name +#define ngx_de_namelen(dir) (dir)->namelen ngx_int_t ngx_de_info(u_char *name, ngx_dir_t *dir); #define ngx_de_info_n "dummy()" From mdounin at mdounin.ru Thu Jan 12 21:35:25 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:25 +0300 Subject: [PATCH 02 of 12] Win32: non-ASCII names support in "include" with wildcards In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1673548899 -10800 # Thu Jan 12 21:41:39 2023 +0300 # Node ID d05c0adf5890aecc68ce8906ef19ca07502ed071 # Parent 60d845f9505fe1b97c1e04b680523b790e29fdb1 Win32: non-ASCII names support in "include" with wildcards. Notably, ngx_open_glob() now supports opening directories with non-ASCII characters, and pathnames returned by ngx_read_glob() are converted to UTF-8. diff -r 60d845f9505f -r d05c0adf5890 src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:41:30 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:41:39 2023 +0300 @@ -546,14 +546,27 @@ ngx_open_glob(ngx_glob_t *gl) { u_char *p; size_t len; + u_short *u; ngx_err_t err; + u_short utf16[NGX_UTF16_BUFLEN]; - gl->dir = FindFirstFile((const char *) gl->pattern, &gl->finddata); + len = NGX_UTF16_BUFLEN; + u = ngx_utf8_to_utf16(utf16, gl->pattern, &len, 0); + + if (u == NULL) { + return NGX_ERROR; + } + + gl->dir = FindFirstFileW(u, &gl->finddata); if (gl->dir == INVALID_HANDLE_VALUE) { err = ngx_errno; + if (u != utf16) { + ngx_free(u); + } + if ((err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) && gl->test) { @@ -561,6 +574,8 @@ ngx_open_glob(ngx_glob_t *gl) return NGX_OK; } + ngx_set_errno(err); + return NGX_ERROR; } @@ -570,18 +585,10 @@ ngx_open_glob(ngx_glob_t *gl) } } - len = ngx_strlen(gl->finddata.cFileName); - gl->name.len = gl->last + len; - - gl->name.data = ngx_alloc(gl->name.len + 1, gl->log); - if (gl->name.data == NULL) { - return NGX_ERROR; + if (u != utf16) { + ngx_free(u); } - ngx_memcpy(gl->name.data, gl->pattern, gl->last); - ngx_cpystrn(gl->name.data + gl->last, (u_char *) gl->finddata.cFileName, - len + 1); - gl->ready = 1; return NGX_OK; @@ -591,40 +598,25 @@ ngx_open_glob(ngx_glob_t *gl) ngx_int_t ngx_read_glob(ngx_glob_t *gl, ngx_str_t *name) { - size_t len; - ngx_err_t err; + u_char *p; + size_t len; + ngx_err_t err; + u_char utf8[NGX_UTF16_BUFLEN]; if (gl->no_match) { return NGX_DONE; } if (gl->ready) { - *name = gl->name; - gl->ready = 0; - return NGX_OK; + goto convert; } ngx_free(gl->name.data); gl->name.data = NULL; - if (FindNextFile(gl->dir, &gl->finddata) != 0) { - - len = ngx_strlen(gl->finddata.cFileName); - gl->name.len = gl->last + len; - - gl->name.data = ngx_alloc(gl->name.len + 1, gl->log); - if (gl->name.data == NULL) { - return NGX_ERROR; - } - - ngx_memcpy(gl->name.data, gl->pattern, gl->last); - ngx_cpystrn(gl->name.data + gl->last, (u_char *) gl->finddata.cFileName, - len + 1); - - *name = gl->name; - - return NGX_OK; + if (FindNextFileW(gl->dir, &gl->finddata) != 0) { + goto convert; } err = ngx_errno; @@ -637,6 +629,43 @@ ngx_read_glob(ngx_glob_t *gl, ngx_str_t "FindNextFile(%s) failed", gl->pattern); return NGX_ERROR; + +convert: + + len = NGX_UTF16_BUFLEN; + p = ngx_utf16_to_utf8(utf8, gl->finddata.cFileName, &len, NULL); + + if (p == NULL) { + return NGX_ERROR; + } + + gl->name.len = gl->last + len - 1; + + gl->name.data = ngx_alloc(gl->name.len + 1, gl->log); + if (gl->name.data == NULL) { + goto failed; + } + + ngx_memcpy(gl->name.data, gl->pattern, gl->last); + ngx_cpystrn(gl->name.data + gl->last, p, len); + + if (p != utf8) { + ngx_free(p); + } + + *name = gl->name; + + return NGX_OK; + +failed: + + if (p != utf8) { + err = ngx_errno; + ngx_free(p); + ngx_set_errno(err); + } + + return NGX_ERROR; } diff -r 60d845f9505f -r d05c0adf5890 src/os/win32/ngx_files.h --- a/src/os/win32/ngx_files.h Thu Jan 12 21:41:30 2023 +0300 +++ b/src/os/win32/ngx_files.h Thu Jan 12 21:41:39 2023 +0300 @@ -44,7 +44,7 @@ typedef struct { typedef struct { HANDLE dir; - WIN32_FIND_DATA finddata; + WIN32_FIND_DATAW finddata; unsigned ready:1; unsigned test:1; From mdounin at mdounin.ru Thu Jan 12 21:35:27 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:27 +0300 Subject: [PATCH 04 of 12] Win32: non-ASCII directory names support in ngx_create_dir() In-Reply-To: References: Message-ID: <331e40f73047c03455e1.1673559327@1.0.0.127.in-addr.arpa> # HG changeset patch # User Maxim Dounin # Date 1673548958 -10800 # Thu Jan 12 21:42:38 2023 +0300 # Node ID 331e40f73047c03455e10212b2b8701ec5ec35d0 # Parent 7cf820c46860796cff91f53a5d2db669bb5b5a6c Win32: non-ASCII directory names support in ngx_create_dir(). This makes it possible to create directories under prefix with non-ASCII characters, as well as makes it possible to create directories with non-ASCII characters when using the dav module (ticket #1433). To ensure that the dav module operations are restricted similarly to other file operations (in particular, short names are not allowed), the ngx_win32_check_filename() function is used. It improved to support checking of just dirname, and now can be used to check paths when creating files or directories. diff -r 7cf820c46860 -r 331e40f73047 src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:41:56 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:42:38 2023 +0300 @@ -11,8 +11,8 @@ #define NGX_UTF16_BUFLEN 256 -static ngx_int_t ngx_win32_check_filename(u_char *name, u_short *u, - size_t len); +static ngx_int_t ngx_win32_check_filename(u_short *u, size_t len, + ngx_uint_t dirname); static u_short *ngx_utf8_to_utf16(u_short *utf16, u_char *utf8, size_t *len, size_t reserved); static u_char *ngx_utf16_to_utf8(u_char *utf8, u_short *utf16, size_t *len, @@ -41,7 +41,7 @@ ngx_open_file(u_char *name, u_long mode, fd = INVALID_HANDLE_VALUE; if (create == NGX_FILE_OPEN - && ngx_win32_check_filename(name, u, len) != NGX_OK) + && ngx_win32_check_filename(u, len, 0) != NGX_OK) { goto failed; } @@ -281,7 +281,7 @@ ngx_file_info(u_char *file, ngx_file_inf rc = NGX_FILE_ERROR; - if (ngx_win32_check_filename(file, u, len) != NGX_OK) { + if (ngx_win32_check_filename(u, len, 0) != NGX_OK) { goto failed; } @@ -467,7 +467,7 @@ ngx_open_dir(ngx_str_t *name, ngx_dir_t return NGX_ERROR; } - if (ngx_win32_check_filename(name->data, u, len) != NGX_OK) { + if (ngx_win32_check_filename(u, len, 0) != NGX_OK) { goto failed; } @@ -566,6 +566,42 @@ ngx_close_dir(ngx_dir_t *dir) ngx_int_t +ngx_create_dir(u_char *name, ngx_uint_t access) +{ + long rc; + size_t len; + u_short *u; + ngx_err_t err; + u_short utf16[NGX_UTF16_BUFLEN]; + + len = NGX_UTF16_BUFLEN; + u = ngx_utf8_to_utf16(utf16, name, &len, 0); + + if (u == NULL) { + return NGX_FILE_ERROR; + } + + rc = NGX_FILE_ERROR; + + if (ngx_win32_check_filename(u, len, 1) != NGX_OK) { + goto failed; + } + + rc = CreateDirectoryW(u, NULL); + +failed: + + if (u != utf16) { + err = ngx_errno; + ngx_free(u); + ngx_set_errno(err); + } + + return rc; +} + + +ngx_int_t ngx_open_glob(ngx_glob_t *gl) { u_char *p; @@ -779,11 +815,10 @@ ngx_fs_available(u_char *name) static ngx_int_t -ngx_win32_check_filename(u_char *name, u_short *u, size_t len) +ngx_win32_check_filename(u_short *u, size_t len, ngx_uint_t dirname) { - u_char *p, ch; u_long n; - u_short *lu; + u_short *lu, *p, *slash, ch; ngx_err_t err; enum { sw_start = 0, @@ -796,9 +831,14 @@ ngx_win32_check_filename(u_char *name, u /* check for NTFS streams (":"), trailing dots and spaces */ lu = NULL; + slash = NULL; state = sw_start; - for (p = name; *p; p++) { +#if (NGX_SUPPRESS_WARN) + ch = 0; +#endif + + for (p = u; *p; p++) { ch = *p; switch (state) { @@ -812,6 +852,7 @@ ngx_win32_check_filename(u_char *name, u if (ch == '/' || ch == '\\') { state = sw_after_slash; + slash = p; } break; @@ -830,6 +871,7 @@ ngx_win32_check_filename(u_char *name, u if (ch == '/' || ch == '\\') { state = sw_after_slash; + slash = p; break; } @@ -857,6 +899,7 @@ ngx_win32_check_filename(u_char *name, u if (ch == '/' || ch == '\\') { state = sw_after_slash; + slash = p; break; } @@ -885,6 +928,12 @@ ngx_win32_check_filename(u_char *name, u goto invalid; } + if (dirname && slash) { + ch = *slash; + *slash = '\0'; + len = slash - u + 1; + } + /* check if long name match */ lu = malloc(len * 2); @@ -895,6 +944,11 @@ ngx_win32_check_filename(u_char *name, u n = GetLongPathNameW(u, lu, len); if (n == 0) { + + if (dirname && slash && ngx_errno == NGX_ENOENT) { + ngx_set_errno(NGX_ENOPATH); + } + goto failed; } @@ -902,6 +956,10 @@ ngx_win32_check_filename(u_char *name, u goto invalid; } + if (dirname && slash) { + *slash = ch; + } + ngx_free(lu); return NGX_OK; @@ -912,6 +970,10 @@ invalid: failed: + if (dirname && slash) { + *slash = ch; + } + if (lu) { err = ngx_errno; ngx_free(lu); diff -r 7cf820c46860 -r 331e40f73047 src/os/win32/ngx_files.h --- a/src/os/win32/ngx_files.h Thu Jan 12 21:41:56 2023 +0300 +++ b/src/os/win32/ngx_files.h Thu Jan 12 21:42:38 2023 +0300 @@ -202,7 +202,7 @@ ngx_int_t ngx_close_dir(ngx_dir_t *dir); #define ngx_close_dir_n "FindClose()" -#define ngx_create_dir(name, access) CreateDirectory((const char *) name, NULL) +ngx_int_t ngx_create_dir(u_char *name, ngx_uint_t access); #define ngx_create_dir_n "CreateDirectory()" From mdounin at mdounin.ru Thu Jan 12 21:35:28 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:28 +0300 Subject: [PATCH 05 of 12] Win32: non-ASCII directory names support in ngx_delete_dir() In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1673548963 -10800 # Thu Jan 12 21:42:43 2023 +0300 # Node ID fbe7d76fe0398ca674b5d62b4849cd524e35bf89 # Parent 331e40f73047c03455e10212b2b8701ec5ec35d0 Win32: non-ASCII directory names support in ngx_delete_dir(). This makes it possible to delete directories with non-ASCII characters when using the dav module (ticket #1433). diff -r 331e40f73047 -r fbe7d76fe039 src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:42:38 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:42:43 2023 +0300 @@ -602,6 +602,42 @@ failed: ngx_int_t +ngx_delete_dir(u_char *name) +{ + long rc; + size_t len; + u_short *u; + ngx_err_t err; + u_short utf16[NGX_UTF16_BUFLEN]; + + len = NGX_UTF16_BUFLEN; + u = ngx_utf8_to_utf16(utf16, name, &len, 0); + + if (u == NULL) { + return NGX_FILE_ERROR; + } + + rc = NGX_FILE_ERROR; + + if (ngx_win32_check_filename(u, len, 0) != NGX_OK) { + goto failed; + } + + rc = RemoveDirectoryW(u); + +failed: + + if (u != utf16) { + err = ngx_errno; + ngx_free(u); + ngx_set_errno(err); + } + + return rc; +} + + +ngx_int_t ngx_open_glob(ngx_glob_t *gl) { u_char *p; diff -r 331e40f73047 -r fbe7d76fe039 src/os/win32/ngx_files.h --- a/src/os/win32/ngx_files.h Thu Jan 12 21:42:38 2023 +0300 +++ b/src/os/win32/ngx_files.h Thu Jan 12 21:42:43 2023 +0300 @@ -206,7 +206,7 @@ ngx_int_t ngx_create_dir(u_char *name, n #define ngx_create_dir_n "CreateDirectory()" -#define ngx_delete_dir(name) RemoveDirectory((const char *) name) +ngx_int_t ngx_delete_dir(u_char *name); #define ngx_delete_dir_n "RemoveDirectory()" From mdounin at mdounin.ru Thu Jan 12 21:35:29 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:29 +0300 Subject: [PATCH 06 of 12] Win32: reworked ngx_win32_rename_file() to check errors In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1673548968 -10800 # Thu Jan 12 21:42:48 2023 +0300 # Node ID bed2302585d8647df3f8185085588395b1ce7b74 # Parent fbe7d76fe0398ca674b5d62b4849cd524e35bf89 Win32: reworked ngx_win32_rename_file() to check errors. Previously, ngx_win32_rename_file() retried on all errors returned by MoveFile() to a temporary name. It only make sense, however, to retry when the destination file already exists, similarly to the condition when ngx_win32_rename_file() is called. Retrying on other errors is meaningless and might result in an infinite loop. diff -r fbe7d76fe039 -r bed2302585d8 src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:42:43 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:42:48 2023 +0300 @@ -235,10 +235,16 @@ ngx_win32_rename_file(ngx_str_t *from, n break; } - collision = 1; + err = ngx_errno; - ngx_log_error(NGX_LOG_CRIT, log, ngx_errno, + if (err == NGX_EEXIST || err == NGX_EEXIST_FILE) { + collision = 1; + continue; + } + + ngx_log_error(NGX_LOG_CRIT, log, err, "MoveFile() \"%s\" to \"%s\" failed", to->data, name); + goto failed; } if (MoveFile((const char *) from->data, (const char *) to->data) == 0) { @@ -253,6 +259,8 @@ ngx_win32_rename_file(ngx_str_t *from, n "DeleteFile() \"%s\" failed", name); } +failed: + /* mutex_unlock() */ ngx_free(name); From mdounin at mdounin.ru Thu Jan 12 21:35:30 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:30 +0300 Subject: [PATCH 07 of 12] Win32: reworked ngx_win32_rename_file() to use nginx wrappers In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1673548972 -10800 # Thu Jan 12 21:42:52 2023 +0300 # Node ID c86a1405cba4d5c1be6d1bbf3160ba567b3c6f50 # Parent bed2302585d8647df3f8185085588395b1ce7b74 Win32: reworked ngx_win32_rename_file() to use nginx wrappers. This ensures that ngx_win32_rename_file() will support non-ASCII names when supported by the wrappers. Notably, this is used by PUT requests in the dav module when overwriting existing files with non-ASCII names (ticket #1433). diff -r bed2302585d8 -r c86a1405cba4 src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:42:48 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:42:52 2023 +0300 @@ -231,7 +231,7 @@ ngx_win32_rename_file(ngx_str_t *from, n ngx_sprintf(name + to->len, ".%0muA.DELETE%Z", num); - if (MoveFile((const char *) to->data, (const char *) name) != 0) { + if (ngx_rename_file(to->data, name) != NGX_FILE_ERROR) { break; } @@ -247,14 +247,14 @@ ngx_win32_rename_file(ngx_str_t *from, n goto failed; } - if (MoveFile((const char *) from->data, (const char *) to->data) == 0) { + if (ngx_rename_file(from->data, to->data) == NGX_FILE_ERROR) { err = ngx_errno; } else { err = 0; } - if (DeleteFile((const char *) name) == 0) { + if (ngx_delete_file(name) == NGX_FILE_ERROR) { ngx_log_error(NGX_LOG_CRIT, log, ngx_errno, "DeleteFile() \"%s\" failed", name); } From mdounin at mdounin.ru Thu Jan 12 21:35:32 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:32 +0300 Subject: [PATCH 09 of 12] Win32: non-ASCII names support in ngx_rename_file() In-Reply-To: References: Message-ID: <718bf1f036e6e00f09bb.1673559332@1.0.0.127.in-addr.arpa> # HG changeset patch # User Maxim Dounin # Date 1673548982 -10800 # Thu Jan 12 21:43:02 2023 +0300 # Node ID 718bf1f036e6e00f09bb4569b124540ed3153392 # Parent d680dedf87fb861f6265218b5d53b49e951947e0 Win32: non-ASCII names support in ngx_rename_file(). This makes it possible to upload files with non-ASCII characters when using the dav module (ticket #1433). diff -r d680dedf87fb -r 718bf1f036e6 src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:42:58 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:43:02 2023 +0300 @@ -242,6 +242,61 @@ failed: } +ngx_int_t +ngx_rename_file(u_char *from, u_char *to) +{ + long rc; + size_t len; + u_short *fu, *tu; + ngx_err_t err; + u_short utf16f[NGX_UTF16_BUFLEN]; + u_short utf16t[NGX_UTF16_BUFLEN]; + + len = NGX_UTF16_BUFLEN; + fu = ngx_utf8_to_utf16(utf16f, from, &len, 0); + + if (fu == NULL) { + return NGX_FILE_ERROR; + } + + rc = NGX_FILE_ERROR; + tu = NULL; + + if (ngx_win32_check_filename(fu, len, 0) != NGX_OK) { + goto failed; + } + + len = NGX_UTF16_BUFLEN; + tu = ngx_utf8_to_utf16(utf16t, to, &len, 0); + + if (tu == NULL) { + goto failed; + } + + if (ngx_win32_check_filename(tu, len, 1) != NGX_OK) { + goto failed; + } + + rc = MoveFileW(fu, tu); + +failed: + + if (fu != utf16f) { + err = ngx_errno; + ngx_free(fu); + ngx_set_errno(err); + } + + if (tu && tu != utf16t) { + err = ngx_errno; + ngx_free(tu); + ngx_set_errno(err); + } + + return rc; +} + + ngx_err_t ngx_win32_rename_file(ngx_str_t *from, ngx_str_t *to, ngx_log_t *log) { diff -r d680dedf87fb -r 718bf1f036e6 src/os/win32/ngx_files.h --- a/src/os/win32/ngx_files.h Thu Jan 12 21:42:58 2023 +0300 +++ b/src/os/win32/ngx_files.h Thu Jan 12 21:43:02 2023 +0300 @@ -127,7 +127,7 @@ ngx_int_t ngx_delete_file(u_char *name); #define ngx_delete_file_n "DeleteFile()" -#define ngx_rename_file(o, n) MoveFile((const char *) o, (const char *) n) +ngx_int_t ngx_rename_file(u_char *from, u_char *to); #define ngx_rename_file_n "MoveFile()" ngx_err_t ngx_win32_rename_file(ngx_str_t *from, ngx_str_t *to, ngx_log_t *log); From mdounin at mdounin.ru Thu Jan 12 21:35:33 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:33 +0300 Subject: [PATCH 10 of 12] Win32: non-ASCII names support in ngx_open_tempfile() In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1673548994 -10800 # Thu Jan 12 21:43:14 2023 +0300 # Node ID e62c8e9724ba68a698a2c3613edca73fe4e1c4ae # Parent 718bf1f036e6e00f09bb4569b124540ed3153392 Win32: non-ASCII names support in ngx_open_tempfile(). This makes it possible to use temporary directories with non-ASCII characters, either explicitly or via a prefix with non-ASCII characters in it. diff -r 718bf1f036e6 -r e62c8e9724ba src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:43:02 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:43:14 2023 +0300 @@ -62,6 +62,41 @@ failed: } +ngx_fd_t +ngx_open_tempfile(u_char *name, ngx_uint_t persistent, ngx_uint_t access) +{ + size_t len; + u_short *u; + ngx_fd_t fd; + ngx_err_t err; + u_short utf16[NGX_UTF16_BUFLEN]; + + len = NGX_UTF16_BUFLEN; + u = ngx_utf8_to_utf16(utf16, name, &len, 0); + + if (u == NULL) { + return INVALID_HANDLE_VALUE; + } + + fd = CreateFileW(u, + GENERIC_READ|GENERIC_WRITE, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + NULL, + CREATE_NEW, + persistent ? 0: + FILE_ATTRIBUTE_TEMPORARY|FILE_FLAG_DELETE_ON_CLOSE, + NULL); + + if (u != utf16) { + err = ngx_errno; + ngx_free(u); + ngx_set_errno(err); + } + + return fd; +} + + ssize_t ngx_read_file(ngx_file_t *file, u_char *buf, size_t size, off_t offset) { diff -r 718bf1f036e6 -r e62c8e9724ba src/os/win32/ngx_files.h --- a/src/os/win32/ngx_files.h Thu Jan 12 21:43:02 2023 +0300 +++ b/src/os/win32/ngx_files.h Thu Jan 12 21:43:14 2023 +0300 @@ -90,16 +90,8 @@ ngx_fd_t ngx_open_file(u_char *name, u_l #define NGX_FILE_OWNER_ACCESS 0 -#define ngx_open_tempfile(name, persistent, access) \ - CreateFile((const char *) name, \ - GENERIC_READ|GENERIC_WRITE, \ - FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, \ - NULL, \ - CREATE_NEW, \ - persistent ? 0: \ - FILE_ATTRIBUTE_TEMPORARY|FILE_FLAG_DELETE_ON_CLOSE, \ - NULL); - +ngx_fd_t ngx_open_tempfile(u_char *name, ngx_uint_t persistent, + ngx_uint_t access); #define ngx_open_tempfile_n "CreateFile()" From mdounin at mdounin.ru Thu Jan 12 21:35:31 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:31 +0300 Subject: [PATCH 08 of 12] Win32: non-ASCII names support in ngx_delete_file() In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1673548978 -10800 # Thu Jan 12 21:42:58 2023 +0300 # Node ID d680dedf87fb861f6265218b5d53b49e951947e0 # Parent c86a1405cba4d5c1be6d1bbf3160ba567b3c6f50 Win32: non-ASCII names support in ngx_delete_file(). This makes it possible to delete files with non-ASCII characters when using the dav module (ticket #1433). diff -r c86a1405cba4 -r d680dedf87fb src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:42:52 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:42:58 2023 +0300 @@ -206,6 +206,42 @@ ngx_write_console(ngx_fd_t fd, void *buf } +ngx_int_t +ngx_delete_file(u_char *name) +{ + long rc; + size_t len; + u_short *u; + ngx_err_t err; + u_short utf16[NGX_UTF16_BUFLEN]; + + len = NGX_UTF16_BUFLEN; + u = ngx_utf8_to_utf16(utf16, name, &len, 0); + + if (u == NULL) { + return NGX_FILE_ERROR; + } + + rc = NGX_FILE_ERROR; + + if (ngx_win32_check_filename(u, len, 0) != NGX_OK) { + goto failed; + } + + rc = DeleteFileW(u); + +failed: + + if (u != utf16) { + err = ngx_errno; + ngx_free(u); + ngx_set_errno(err); + } + + return rc; +} + + ngx_err_t ngx_win32_rename_file(ngx_str_t *from, ngx_str_t *to, ngx_log_t *log) { diff -r c86a1405cba4 -r d680dedf87fb src/os/win32/ngx_files.h --- a/src/os/win32/ngx_files.h Thu Jan 12 21:42:52 2023 +0300 +++ b/src/os/win32/ngx_files.h Thu Jan 12 21:42:58 2023 +0300 @@ -123,7 +123,7 @@ ssize_t ngx_write_console(ngx_fd_t fd, v #define NGX_LINEFEED CRLF -#define ngx_delete_file(name) DeleteFile((const char *) name) +ngx_int_t ngx_delete_file(u_char *name); #define ngx_delete_file_n "DeleteFile()" From mdounin at mdounin.ru Thu Jan 12 21:35:35 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:35 +0300 Subject: [PATCH 12 of 12] Win32: non-ASCII names in ngx_fs_bsize(), ngx_fs_available() In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1673549053 -10800 # Thu Jan 12 21:44:13 2023 +0300 # Node ID c63770d2ae19a5a84f3521e336ca2092879dd855 # Parent be7eb9ec28dcbfdfd2e850befc8d051c0e4d46fd Win32: non-ASCII names in ngx_fs_bsize(), ngx_fs_available(). This fixes potentially incorrect cache size calculations and non-working "min_free" when using cache in directories with non-ASCII names. diff -r be7eb9ec28dc -r c63770d2ae19 src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:43:30 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:44:13 2023 +0300 @@ -955,12 +955,31 @@ ngx_directio_off(ngx_fd_t fd) size_t ngx_fs_bsize(u_char *name) { - u_long sc, bs, nfree, ncl; + u_long sc, bs, nfree, ncl; + size_t len; + u_short *u; + u_short utf16[NGX_UTF16_BUFLEN]; + + len = NGX_UTF16_BUFLEN; + u = ngx_utf8_to_utf16(utf16, name, &len, 0); + + if (u == NULL) { + return 512; + } - if (GetDiskFreeSpace((const char *) name, &sc, &bs, &nfree, &ncl) == 0) { + if (GetDiskFreeSpaceW(u, &sc, &bs, &nfree, &ncl) == 0) { + + if (u != utf16) { + ngx_free(u); + } + return 512; } + if (u != utf16) { + ngx_free(u); + } + return sc * bs; } @@ -968,12 +987,31 @@ ngx_fs_bsize(u_char *name) off_t ngx_fs_available(u_char *name) { - ULARGE_INTEGER navail; + size_t len; + u_short *u; + ULARGE_INTEGER navail; + u_short utf16[NGX_UTF16_BUFLEN]; + + len = NGX_UTF16_BUFLEN; + u = ngx_utf8_to_utf16(utf16, name, &len, 0); + + if (u == NULL) { + return NGX_MAX_OFF_T_VALUE; + } - if (GetDiskFreeSpaceEx((const char *) name, &navail, NULL, NULL) == 0) { + if (GetDiskFreeSpaceExW(u, &navail, NULL, NULL) == 0) { + + if (u != utf16) { + ngx_free(u); + } + return NGX_MAX_OFF_T_VALUE; } + if (u != utf16) { + ngx_free(u); + } + return (off_t) navail.QuadPart; } From mdounin at mdounin.ru Thu Jan 12 21:35:34 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Fri, 13 Jan 2023 00:35:34 +0300 Subject: [PATCH 11 of 12] Win32: fixed ngx_fs_bsize() for symlinks In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1673549010 -10800 # Thu Jan 12 21:43:30 2023 +0300 # Node ID be7eb9ec28dcbfdfd2e850befc8d051c0e4d46fd # Parent e62c8e9724ba68a698a2c3613edca73fe4e1c4ae Win32: fixed ngx_fs_bsize() for symlinks. Just a drive letter might not correctly represent file system being used, notably when using symlinks (as created by "mklink /d"). As such, instead of calling GetDiskFreeSpace() with just a drive letter, we now always use GetDiskFreeSpace() with full path. diff -r e62c8e9724ba -r be7eb9ec28dc src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Jan 12 21:43:14 2023 +0300 +++ b/src/os/win32/ngx_files.c Thu Jan 12 21:43:30 2023 +0300 @@ -955,14 +955,8 @@ ngx_directio_off(ngx_fd_t fd) size_t ngx_fs_bsize(u_char *name) { - u_char root[4]; u_long sc, bs, nfree, ncl; - if (name[2] == ':') { - ngx_cpystrn(root, name, 4); - name = root; - } - if (GetDiskFreeSpace((const char *) name, &sc, &bs, &nfree, &ncl) == 0) { return 512; } From mdounin at mdounin.ru Fri Jan 13 01:30:15 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Fri, 13 Jan 2023 04:30:15 +0300 Subject: [PATCH 1 of 4] Win32: removed unneeded wildcard in NGX_CC_NAME test for msvc In-Reply-To: References: Message-ID: Hello! On Tue, Dec 20, 2022 at 04:30:15PM +0300, Maxim Dounin wrote: > # HG changeset patch > # User Maxim Dounin > # Date 1671541071 -10800 > # Tue Dec 20 15:57:51 2022 +0300 > # Node ID f5d9c24fb4ac2a6b82b9d842b88978a329690138 > # Parent 2af1287d2da744335932f6dca345618f7b80d1c1 > Win32: removed unneeded wildcard in NGX_CC_NAME test for msvc. > > Wildcards for msvc in NGX_CC_NAME tests are not needed since 78f8ac479735. > > diff -r 2af1287d2da7 -r f5d9c24fb4ac auto/cc/conf > --- a/auto/cc/conf Sun Dec 18 21:29:02 2022 +0300 > +++ b/auto/cc/conf Tue Dec 20 15:57:51 2022 +0300 > @@ -117,7 +117,7 @@ else > . auto/cc/acc > ;; > > - msvc*) > + msvc) > # MSVC++ 6.0 SP2, MSVC++ Toolkit 2003 > > . auto/cc/msvc Ping. -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Fri Jan 13 01:31:00 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Fri, 13 Jan 2023 04:31:00 +0300 Subject: [PATCH] Added warning about redefinition of listen socket protocol options In-Reply-To: References: Message-ID: Hello! On Sat, Dec 31, 2022 at 05:35:59PM +0300, Maxim Dounin wrote: > # HG changeset patch > # User Maxim Dounin > # Date 1672497248 -10800 > # Sat Dec 31 17:34:08 2022 +0300 > # Node ID c215d5cf25732ece1819cf1cd48ebb480bb642c7 > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > Added warning about redefinition of listen socket protocol options. > > The "listen" directive in the http module can be used multiple times > in different server blocks. Originally, it was supposed to be specified > once with various socket options, and without any parameters in virtual > server blocks. For example: > > server { listen 80 backlog=1024; server_name foo; ... } > server { listen 80; server_name bar; ... } > server { listen 80; server_name bazz; ... } > > The address part of the syntax ("address[:port]" / "port" / "unix:path") > uniquely identifies the listening socket, and therefore is enough for > name-based virtual servers (to let nginx know that the virtual server > accepts requests on the listening socket in question). > > To ensure that listening options do not conflict between virtual servers, > they were allowed only once. For example, the following configuration > will be rejected ("duplicate listen options for 0.0.0.0:80 in ..."): > > server { listen 80 backlog=1024; server_name foo; ... } > server { listen 80 backlog=512; server_name bar; ... } > > At some point it was, however, noticed, that it is sometimes convenient > to repeat some options for clarity. In nginx 0.8.51 the "ssl" parameter > was allowed to be specified multiple times, e.g.: > > server { listen 443 ssl backlog=1024; server_name foo; ... } > server { listen 443 ssl; server_name bar; ... } > server { listen 443 ssl; server_name bazz; ... } > > This approach makes configuration more readable, since SSL sockets are > immediately visible in the configuration. If this is not needed, just the > address can still be used. > > Later, additional protocol-specific options similar to "ssl" were > introduced, notably "http2" and "proxy_protocol". With these options, > one can write: > > server { listen 443 ssl backlog=1024; server_name foo; ... } > server { listen 443 http2; server_name bar; ... } > server { listen 443 proxy_protocol; server_name bazz; ... } > > The resulting socket will use ssl, http2, and proxy_protocol, but this > is not really obvious from the configuration. > > To ensure such misleading configurations are not allowed, nginx now > warns as long as the "listen" directive is used with options different > from the options previously used if these are potentially confusing. > > In particular, the following configurations are allowed: > > server { listen 8401 ssl backlog=1024; server_name foo; } > server { listen 8401 ssl; server_name bar; } > server { listen 8401 ssl; server_name bazz; } > > server { listen 8402 ssl http2 backlog=1024; server_name foo; } > server { listen 8402 ssl; server_name bar; } > server { listen 8402 ssl; server_name bazz; } > > server { listen 8403 ssl; server_name bar; } > server { listen 8403 ssl; server_name bazz; } > server { listen 8403 ssl http2; server_name foo; } > > server { listen 8404 ssl http2 backlog=1024; server_name foo; } > server { listen 8404 http2; server_name bar; } > server { listen 8404 http2; server_name bazz; } > > server { listen 8405 ssl http2 backlog=1024; server_name foo; } > server { listen 8405 ssl http2; server_name bar; } > server { listen 8405 ssl http2; server_name bazz; } > > server { listen 8406 ssl; server_name foo; } > server { listen 8406; server_name bar; } > server { listen 8406; server_name bazz; } > > And the following configurations will generate warnings: > > server { listen 8501 ssl http2 backlog=1024; server_name foo; } > server { listen 8501 http2; server_name bar; } > server { listen 8501 ssl; server_name bazz; } > > server { listen 8502 backlog=1024; server_name foo; } > server { listen 8502 ssl; server_name bar; } > > server { listen 8503 ssl; server_name foo; } > server { listen 8503 http2; server_name bar; } > > server { listen 8504 ssl; server_name foo; } > server { listen 8504 http2; server_name bar; } > server { listen 8504 proxy_protocol; server_name bazz; } > > server { listen 8505 ssl http2 proxy_protocol; server_name foo; } > server { listen 8505 ssl http2; server_name bar; } > server { listen 8505 ssl; server_name bazz; } > > server { listen 8506 ssl http2; server_name foo; } > server { listen 8506 ssl; server_name bar; } > server { listen 8506; server_name bazz; } > > server { listen 8507 ssl; server_name bar; } > server { listen 8507; server_name bazz; } > server { listen 8507 ssl http2; server_name foo; } > > server { listen 8508 ssl; server_name bar; } > server { listen 8508; server_name bazz; } > server { listen 8508 ssl backlog=1024; server_name foo; } > > server { listen 8509; server_name bazz; } > server { listen 8509 ssl; server_name bar; } > server { listen 8509 ssl backlog=1024; server_name foo; } > > The basic idea is that at most two sets of protocol options are allowed: > the main one (with socket options, if any), and a shorter one, with options > being a subset of the main options, repeated for clarity. As long as the > shorter set of protocol options is used, all listen directives except the > main one should use it. > > diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c > --- a/src/http/ngx_http.c > +++ b/src/http/ngx_http.c > @@ -1228,7 +1228,8 @@ static ngx_int_t > ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, > ngx_http_conf_port_t *port, ngx_http_listen_opt_t *lsopt) > { > - ngx_uint_t i, default_server, proxy_protocol; > + ngx_uint_t i, default_server, proxy_protocol, > + protocols, protocols_prev; > ngx_http_conf_addr_t *addr; > #if (NGX_HTTP_SSL) > ngx_uint_t ssl; > @@ -1264,12 +1265,18 @@ ngx_http_add_addresses(ngx_conf_t *cf, n > default_server = addr[i].opt.default_server; > > proxy_protocol = lsopt->proxy_protocol || addr[i].opt.proxy_protocol; > + protocols = lsopt->proxy_protocol; > + protocols_prev = addr[i].opt.proxy_protocol; > > #if (NGX_HTTP_SSL) > ssl = lsopt->ssl || addr[i].opt.ssl; > + protocols |= lsopt->ssl << 1; > + protocols_prev |= addr[i].opt.ssl << 1; > #endif > #if (NGX_HTTP_V2) > http2 = lsopt->http2 || addr[i].opt.http2; > + protocols |= lsopt->http2 << 2; > + protocols_prev |= addr[i].opt.http2 << 2; > #endif > > if (lsopt->set) { > @@ -1299,6 +1306,55 @@ ngx_http_add_addresses(ngx_conf_t *cf, n > addr[i].default_server = cscf; > } > > + /* check for conflicting protocol options */ > + > + if ((protocols | protocols_prev) != protocols_prev) { > + > + /* options added */ > + > + if ((addr[i].opt.set && !lsopt->set) > + || addr[i].protocols_changed > + || (protocols | protocols_prev) != protocols) > + { > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > + "protocol options redefined for %V", > + &addr[i].opt.addr_text); > + } > + > + addr[i].protocols = protocols_prev; > + addr[i].protocols_set = 1; > + addr[i].protocols_changed = 1; > + > + } else if ((protocols_prev | protocols) != protocols) { > + > + /* options removed */ > + > + if (lsopt->set > + || (addr[i].protocols_set && protocols != addr[i].protocols)) > + { > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > + "protocol options redefined for %V", > + &addr[i].opt.addr_text); > + } > + > + addr[i].protocols = protocols; > + addr[i].protocols_set = 1; > + addr[i].protocols_changed = 1; > + > + } else { > + > + /* the same options */ > + > + if (lsopt->set && addr[i].protocols_changed) { > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > + "protocol options redefined for %V", > + &addr[i].opt.addr_text); > + } > + > + addr[i].protocols = protocols; > + addr[i].protocols_set = 1; > + } > + > addr[i].opt.default_server = default_server; > addr[i].opt.proxy_protocol = proxy_protocol; > #if (NGX_HTTP_SSL) > @@ -1355,6 +1411,9 @@ ngx_http_add_address(ngx_conf_t *cf, ngx > } > > addr->opt = *lsopt; > + addr->protocols = 0; > + addr->protocols_set = 0; > + addr->protocols_changed = 0; > addr->hash.buckets = NULL; > addr->hash.size = 0; > addr->wc_head = NULL; > diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h > --- a/src/http/ngx_http_core_module.h > +++ b/src/http/ngx_http_core_module.h > @@ -274,6 +274,10 @@ typedef struct { > typedef struct { > ngx_http_listen_opt_t opt; > > + unsigned protocols:3; > + unsigned protocols_set:1; > + unsigned protocols_changed:1; > + > ngx_hash_t hash; > ngx_hash_wildcard_t *wc_head; > ngx_hash_wildcard_t *wc_tail; Ping. -- Maxim Dounin http://mdounin.ru/ From liebman at zod.com Fri Jan 13 12:52:43 2023 From: liebman at zod.com (Christopher Liebman) Date: Fri, 13 Jan 2023 04:52:43 -0800 Subject: [PATCH] limit_req_status: allow status response to be as low as 200 In-Reply-To: References: Message-ID: Thanks! On Thu, Jan 12, 2023 at 1:36 PM J Carter wrote: > It's trivial to do with error page OP as mentioned - an example: > > limit_req_status 598; > limit_req_zone $binary_remote_addr zone=a:1m rate=1r/m; > > server { > limit_req zone=a nodelay; > > error_page 598 = @send-204; > > location / { > ... > } > > ... > > location @send-204 { > return 204; > } > } > > error_page's '=' can also be set to 204, and the named location contain > nothing at all (whatever you prefer). > > On 12/01/2023 17:11, Maxim Dounin wrote: > > Hello! > > > > On Thu, Jan 12, 2023 at 05:16:21AM -0800, Christopher Liebman wrote: > > > >> Not with 204. > >> This works quite well with a partner that has an aversion to errors when > >> they run over the limit: > >> limit_req_status 204; > > Indeed, 204 happens to be one of the two 2xx codes which can be > > returned directly, as they are handled in > > ngx_http_finalize_request() to facilitate simpler > > code in the dav module. This is not what your patch enables > > though. For all other codes, except 204 and 201 mentioned above, > > just returning them will simply break things. > > > > As already suggested, proper behaviour for all the codes can be > > easily configured by using the "error_page" directive. > > > > Hope this helps. > > > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel > -------------- next part -------------- An HTML attachment was scrubbed... URL: From mdounin at mdounin.ru Sat Jan 14 00:01:51 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sat, 14 Jan 2023 03:01:51 +0300 Subject: [PATCH] Gzip static: ranges support (ticket #2349) In-Reply-To: References: Message-ID: Hello! On Tue, Jan 03, 2023 at 05:48:36AM +0300, Maxim Dounin wrote: > # HG changeset patch > # User Maxim Dounin > # Date 1672713976 -10800 > # Tue Jan 03 05:46:16 2023 +0300 > # Node ID e0688b4494f02dcf6feebf0c73e02749bd7de381 > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > Gzip static: ranges support (ticket #2349). > > In contrast to on-the-fly gzipping with gzip filter, static gzipped > representation as returned by gzip_static is persistent, and therefore > the same binary representation is available for future requests, making > it possible to use range requests. > > Further, if a gzipped representation is re-generated with different > compression settings, it is expected to result in different ETag and > different size reported in the Content-Range header, making it possible > to safely use range requests anyway. > > As such, ranges are now allowed for files returned by gzip_static. > > diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c > --- a/src/http/modules/ngx_http_gzip_static_module.c > +++ b/src/http/modules/ngx_http_gzip_static_module.c > @@ -247,6 +247,8 @@ ngx_http_gzip_static_handler(ngx_http_re > ngx_str_set(&h->value, "gzip"); > r->headers_out.content_encoding = h; > > + r->allow_ranges = 1; > + > /* we need to allocate all before the header would be sent */ > > b = ngx_calloc_buf(r->pool); Ping. -- Maxim Dounin http://mdounin.ru/ From jmheisz at gmail.com Sat Jan 14 04:19:14 2023 From: jmheisz at gmail.com (Jeff Heisz) Date: Fri, 13 Jan 2023 23:19:14 -0500 Subject: Sending empty response from http_upstream Message-ID: Hello! I've been trying to debug an issue with a custom nginx module, which is working 'properly' with desktop browsers but failing on Apple browsers with an 'invalid response' error message. I don't know exactly what it doesn't like (so much fun debugging on i-devices) but I am seeing one specific oddity when tracing the response from nginx through the desktop debugging console that might be the cause. My module communicates with an upstream manager process that signals the kind of response to be sent along with associated data. The response that has the issue is when the manager signals a redirect instead of a content response. The module extracts the data from the upstream buffer and pushes a req->headers_out.location header record with the URL from the upstream. It then sets up the response as: upstr->headers_in.content_length_n = 0; // (tried this) ngx_http_clear_content_length(req); upstr->headers_in.status_n = NGX_HTTP_MOVED_TEMPORARILY; upstr->state->status = NGX_HTTP_MOVED_TEMPORARILY; upstr->buffer.pos += url_len; req->header_only = 1; upstr->keepalive = 1; Basically, a response of zero length and a 302 response code, advancing the upstream buffer position to the end of the URL (no additional data in the upstream buffer to be processed). However, when I examine the response through the debugger, I see the appropriate location header but the content-length is always 32677! Not zero. Attempting to look at the response shows no data. This is highly suspicious as to why an error is occuring on the i-device. I should also mention that the upstream response with the redirect URL is only 489 bytes, so that's not the source of the value. What I don't understand is where that content-length is coming from. Other cases where I set a non-zero length and have data in the upstream buffer works properly. But for the zero case it doesn't send any content but sends this non-zero length. From the code above you can see that I tried deleting the response content length information in case it was already in place from somewhere but that had no effect. Any thoughts? I've tried doing some tracing in the nginx core code itself but so far have not been able to see where this length is originating from. Many thanks for any input, jmh From mdounin at mdounin.ru Sat Jan 14 18:15:10 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sat, 14 Jan 2023 21:15:10 +0300 Subject: Sending empty response from http_upstream In-Reply-To: References: Message-ID: Hello! On Fri, Jan 13, 2023 at 11:19:14PM -0500, Jeff Heisz wrote: > Hello! > > I've been trying to debug an issue with a custom nginx module, which > is working 'properly' with desktop browsers but failing on Apple > browsers with an 'invalid response' error message. > > I don't know exactly what it doesn't like (so much fun debugging on > i-devices) but I am seeing one specific oddity when tracing the > response from nginx through the desktop debugging console that might > be the cause. > > My module communicates with an upstream manager process that signals > the kind of response to be sent along with associated data. The > response that has the issue is when the manager signals a redirect > instead of a content response. > > The module extracts the data from the upstream buffer and pushes a > req->headers_out.location header record with the URL from the > upstream. It then sets up the response as: > > upstr->headers_in.content_length_n = 0; > // (tried this) ngx_http_clear_content_length(req); > upstr->headers_in.status_n = NGX_HTTP_MOVED_TEMPORARILY; > upstr->state->status = NGX_HTTP_MOVED_TEMPORARILY; > upstr->buffer.pos += url_len; > req->header_only = 1; > upstr->keepalive = 1; At least "req->header_only = 1;" looks problematic. In general, you shouldn't touch request properties from an upstream protocol module. Instead, you should set appropriate upstream's header_in data, so the upstream module will copy relevant information when needed (and if needed). Further, req->header_only is almost always wrong when you are trying to set it yourself. It is expected to be set for responses without response body, such as responses to HEAD requests and 304/204 responses, but this is something nginx header filter will do for you. For other responses, such as 302 in your code, it's incorrect: these are responses with body, and the response body is expected to be sent. With content length set to 0 things might appear to work in some simple cases, but will break nicely if the content length is removed for some reason (e.g., due to potential response modifications by some filter, such as SSI or gzip). That is, things might appear to work correctly with req->header_only explicitly set to 1, but most likely they aren't. Further, the fact it's there suggests it was added to "fix" something, and this in turn might indicate you are doing something else incorrectly. > Basically, a response of zero length and a 302 response code, > advancing the upstream buffer position to the end of the URL (no > additional data in the upstream buffer to be processed). > > However, when I examine the response through the debugger, I see the > appropriate location header but the content-length is always 32677! > Not zero. Attempting to look at the response shows no data. This is > highly suspicious as to why an error is occuring on the i-device. I > should also mention that the upstream response with the redirect URL > is only 489 bytes, so that's not the source of the value. > > What I don't understand is where that content-length is coming from. > Other cases where I set a non-zero length and have data in the > upstream buffer works properly. But for the zero case it doesn't send > any content but sends this non-zero length. From the code above you > can see that I tried deleting the response content length information > in case it was already in place from somewhere but that had no effect. > > Any thoughts? I've tried doing some tracing in the nginx core code > itself but so far have not been able to see where this length is > originating from. It is hard to say anything beyond the above without seeing the code, but first thing I would recommend to do is to look into nginx debug logs, see here for details: http://nginx.org/en/docs/debugging_log.html Debug logs contain detailed information about request processing, including all the headers sent, and should be helpful to understand what's going on. -- Maxim Dounin http://mdounin.ru/ From jmheisz at gmail.com Sun Jan 15 03:28:29 2023 From: jmheisz at gmail.com (Jeff Heisz) Date: Sat, 14 Jan 2023 22:28:29 -0500 Subject: Sending empty response from http_upstream In-Reply-To: References: Message-ID: Ah, ok. I'm not sure where I got that line to mark header_only from, probably looking at some other code. Once I commented that out, I discovered where the content length was coming from, I had an error page for the 302 response that I hadn't realized I'd put in there and of course it makes sense in this case that there was a response length but no body because I had forced the flag. Code adjusted, no more invalid response errors on i-devices! Thanks for the help Maxim. jmh On Fri, Jan 13, 2023 at 11:19 PM Jeff Heisz wrote: > > Hello! > > I've been trying to debug an issue with a custom nginx module, which > is working 'properly' with desktop browsers but failing on Apple > browsers with an 'invalid response' error message. > > I don't know exactly what it doesn't like (so much fun debugging on > i-devices) but I am seeing one specific oddity when tracing the > response from nginx through the desktop debugging console that might > be the cause. > > My module communicates with an upstream manager process that signals > the kind of response to be sent along with associated data. The > response that has the issue is when the manager signals a redirect > instead of a content response. > > The module extracts the data from the upstream buffer and pushes a > req->headers_out.location header record with the URL from the > upstream. It then sets up the response as: > > upstr->headers_in.content_length_n = 0; > // (tried this) ngx_http_clear_content_length(req); > upstr->headers_in.status_n = NGX_HTTP_MOVED_TEMPORARILY; > upstr->state->status = NGX_HTTP_MOVED_TEMPORARILY; > upstr->buffer.pos += url_len; > req->header_only = 1; > upstr->keepalive = 1; > > Basically, a response of zero length and a 302 response code, > advancing the upstream buffer position to the end of the URL (no > additional data in the upstream buffer to be processed). > > However, when I examine the response through the debugger, I see the > appropriate location header but the content-length is always 32677! > Not zero. Attempting to look at the response shows no data. This is > highly suspicious as to why an error is occuring on the i-device. I > should also mention that the upstream response with the redirect URL > is only 489 bytes, so that's not the source of the value. > > What I don't understand is where that content-length is coming from. > Other cases where I set a non-zero length and have data in the > upstream buffer works properly. But for the zero case it doesn't send > any content but sends this non-zero length. From the code above you > can see that I tried deleting the response content length information > in case it was already in place from somewhere but that had no effect. > > Any thoughts? I've tried doing some tracing in the nginx core code > itself but so far have not been able to see where this length is > originating from. > > Many thanks for any input, > > jmh From arut at nginx.com Mon Jan 16 12:37:34 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 16 Jan 2023 16:37:34 +0400 Subject: [PATCH 1 of 6] QUIC: ignore server address while looking up a connection In-Reply-To: References: <1038d7300c29eea02b47.1670578727@ip-10-1-18-114.eu-central-1.compute.internal> Message-ID: <20230116123734.4c2s2qekbh2o6cln@N00W24XTQX> Hi, On Tue, Dec 13, 2022 at 08:49:18PM +0300, Maxim Dounin wrote: > Hello! > > On Fri, Dec 09, 2022 at 09:38:47AM +0000, Roman Arutyunyan wrote: > > > # HG changeset patch > > # User Roman Arutyunyan > > # Date 1670322119 0 > > # Tue Dec 06 10:21:59 2022 +0000 > > # Branch quic > > # Node ID 1038d7300c29eea02b47eac3f205e293b1e55f5b > > # Parent b87a0dbc1150f415def5bc1e1f00d02b33519026 > > QUIC: ignore server address while looking up a connection. > > > > The server connection check was copied from the common UDP code in c2f5d79cde64. > > In QUIC it does not make much sense though. Technically client is not allowed > > to migrate to a different server address. However, migrating withing a single > > wildcard listening does not seem to affect anything. [..] > As a trivial example, one can block packets to a particular server > address on a firewall (in an attempt to stop an attack), with > something like "block from any to 192.0.2.1", assuming it will > stop traffic to the server in question. Still, with the proposed > change, it will be possible to access resources with a previously > established QUIC connection as long as the attacker knows other IP > addresses used on the same physical server. This indeed makes sense. I will remove this patch from the series. -- Roman Arutyunyan From pluknet at nginx.com Mon Jan 16 13:27:04 2023 From: pluknet at nginx.com (=?iso-8859-1?q?Sergey_Kandaurov?=) Date: Mon, 16 Jan 2023 17:27:04 +0400 Subject: [PATCH] QUIC: defer setting the active flag for client stream events Message-ID: # HG changeset patch # User Sergey Kandaurov # Date 1673875616 -14400 # Mon Jan 16 17:26:56 2023 +0400 # Branch quic # Node ID f7c7cabe232898db5b16a142163de71964cebcfd # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 QUIC: defer setting the active flag for client stream events. Specifically, now it is kept unset until streams are initialized. Notably, this unbreaks OCSP with client certificates after 35e27117b593. Previously, the read event could be posted prematurely in ngx_quic_set_event() e.g., as part of handling a STREAM frame. diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c --- a/src/event/quic/ngx_event_quic_streams.c +++ b/src/event/quic/ngx_event_quic_streams.c @@ -106,6 +106,13 @@ ngx_quic_open_stream(ngx_connection_t *c return NULL; } + nqs->connection->write->active = 1; + nqs->connection->write->ready = 1; + + if (!bidi) { + nqs->connection->read->active = 1; + } + return nqs->connection; } @@ -534,6 +541,13 @@ ngx_quic_init_stream_handler(ngx_event_t ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic init stream"); + if ((qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0) { + c->write->active = 1; + c->write->ready = 1; + } + + c->read->active = 1; + ngx_queue_remove(&qs->queue); c->listening->handler(c); @@ -704,19 +718,6 @@ ngx_quic_create_stream(ngx_connection_t log->connection = sc->number; - if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0 - || (id & NGX_QUIC_STREAM_SERVER_INITIATED)) - { - sc->write->active = 1; - sc->write->ready = 1; - } - - if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0 - || (id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0) - { - sc->read->active = 1; - } - if (id & NGX_QUIC_STREAM_UNIDIRECTIONAL) { if (id & NGX_QUIC_STREAM_SERVER_INITIATED) { qs->send_max_data = qc->ctp.initial_max_stream_data_uni; From pluknet at nginx.com Mon Jan 16 14:37:16 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Mon, 16 Jan 2023 18:37:16 +0400 Subject: [PATCH] QUIC: relocated ngx_quic_init_streams() for 0-RTT In-Reply-To: <5a09008e93a3842dc5a3.1673357454@arut-laptop> References: <5a09008e93a3842dc5a3.1673357454@arut-laptop> Message-ID: <50358A91-FC18-4B44-A8AB-EC33B4999DE0@nginx.com> > On 10 Jan 2023, at 17:30, Roman Arutyunyan wrote: > > # HG changeset patch > # User Roman Arutyunyan > # Date 1673357050 -14400 > # Tue Jan 10 17:24:10 2023 +0400 > # Branch quic > # Node ID 5a09008e93a3842dc5a372e39617097893293ddf > # Parent 7dac46ecb27d3f488d40b28cd8d07f884046e171 > QUIC: relocated ngx_quic_init_streams() for 0-RTT. > > Previously, streams were initialized in early keys handler. However, client > transport parameters may not be available by then. This happens, for example, > when using QuicTLS. Now streams are initialized in ngx_quic_crypto_input() > after calling SSL_do_handshake() for both 0-RTT and 1-RTT. > > diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c > --- a/src/event/quic/ngx_event_quic_ssl.c > +++ b/src/event/quic/ngx_event_quic_ssl.c > @@ -67,12 +67,6 @@ ngx_quic_set_read_secret(ngx_ssl_conn_t > return 0; > } > > - if (level == ssl_encryption_early_data) { > - if (ngx_quic_init_streams(c) != NGX_OK) { > - return 0; > - } > - } > - > return 1; > } > > @@ -138,10 +132,6 @@ ngx_quic_set_encryption_secrets(ngx_ssl_ > } > > if (level == ssl_encryption_early_data) { > - if (ngx_quic_init_streams(c) != NGX_OK) { > - return 0; > - } > - > return 1; > } > > @@ -455,11 +445,17 @@ ngx_quic_crypto_input(ngx_connection_t * > qc->error_reason = "handshake failed"; > return NGX_ERROR; > } > - > - return NGX_OK; > } > > - if (SSL_in_init(ssl_conn)) { > + if (n <= 0 || SSL_in_init(ssl_conn)) { > + if (ngx_quic_keys_available(qc->keys, ssl_encryption_early_data) > + && qc->client_tp_done) > + { > + if (ngx_quic_init_streams(c) != NGX_OK) { > + return NGX_ERROR; > + } > + } > + > return NGX_OK; > } > Looks good. -- Sergey Kandaurov From arut at nginx.com Tue Jan 17 09:54:48 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Tue, 17 Jan 2023 13:54:48 +0400 Subject: [PATCH 4 of 6] QUIC: never disable QUIC socket events In-Reply-To: References: Message-ID: <20230117095448.sv4mybjxbovfwgzj@N00W24XTQX> Hi, On Tue, Dec 13, 2022 at 08:57:12PM +0300, Maxim Dounin wrote: > Hello! > > On Fri, Dec 09, 2022 at 09:38:50AM +0000, Roman Arutyunyan wrote: > > > # HG changeset patch > > # User Roman Arutyunyan > > # Date 1670256830 0 > > # Mon Dec 05 16:13:50 2022 +0000 > > # Branch quic > > # Node ID de8bcaea559d151f5945d0a2e06c61b56a26a52b > > # Parent b5c30f16ec8ba3ace2f58d77d294d9b355bf3267 > > QUIC: never disable QUIC socket events. > > > > Unlike TCP accept(), current QUIC implementation does not require new file > > descriptors for new clients. Also, it does not work with accept mutex since > > it normally requires reuseport option. > > > > diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c > > --- a/src/event/ngx_event_accept.c > > +++ b/src/event/ngx_event_accept.c > > @@ -416,6 +416,12 @@ ngx_disable_accept_events(ngx_cycle_t *c > > > > #endif > > > > +#if (NGX_QUIC) > > + if (ls[i].quic) { > > + continue; > > + } > > +#endif > > + > > As long as the reuseport option is used, this should happen > automatically. If it's not, it might be a bad idea to do not > disable accepting new QUIC connections, given that QUIC > connections can still use file descriptors for other tasks, such > as opening files and connecting to backends. > > On the other hand, existing QUIC implementation cannot receive > packets from previously created connections if socket events are > disabled. And this might be a good reason for the change. > If this is the reason, shouldn't it be in the commit log? Exactly. With current QUIC implementation, QUIC listen sockets aren't used only for "listening", but also for receiving QUIC packets for existing connections. > Also, it looks like with client sockets, as introduced later in > this patch series, it would be appropriate to actually disable > events on QUIC listening sockets. IMHO, even when client sockets are committed, the current approach will still be available as an option. Client sockets approach has a number of issues, which the current implementation does not have. > > if (ngx_del_event(c->read, NGX_READ_EVENT, NGX_DISABLE_EVENT) > > == NGX_ERROR) > > { Also, it looks like the problem is not specific to QUIC, but also exists for regular Stream/UDP and should be addressed separately as a fix for mainline. Probably we need a listen flag which would signal if the listen socket should never be disabled, or just a more sophisticated condition in ngx_disable_accept_events(). Anyway, I'll remove the patch from this series. > > > > _______________________________________________ > > nginx-devel mailing list -- nginx-devel at nginx.org > > To unsubscribe send an email to nginx-devel-leave at nginx.org > > -- > Maxim Dounin > http://mdounin.ru/ > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel -- Roman Arutyunyan From arut at nginx.com Tue Jan 17 10:44:15 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Tue, 17 Jan 2023 14:44:15 +0400 Subject: [PATCH] QUIC: defer setting the active flag for client stream events In-Reply-To: References: Message-ID: <4EB02BE1-CF83-4ED5-9282-FA5D26651ADE@nginx.com> > On 16 Jan 2023, at 17:27, Sergey Kandaurov wrote: > > # HG changeset patch > # User Sergey Kandaurov > # Date 1673875616 -14400 > # Mon Jan 16 17:26:56 2023 +0400 > # Branch quic > # Node ID f7c7cabe232898db5b16a142163de71964cebcfd > # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 > QUIC: defer setting the active flag for client stream events. > > Specifically, now it is kept unset until streams are initialized. > Notably, this unbreaks OCSP with client certificates after 35e27117b593. > Previously, the read event could be posted prematurely in ngx_quic_set_event() > e.g., as part of handling a STREAM frame. > > diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c > --- a/src/event/quic/ngx_event_quic_streams.c > +++ b/src/event/quic/ngx_event_quic_streams.c > @@ -106,6 +106,13 @@ ngx_quic_open_stream(ngx_connection_t *c > return NULL; > } > > + nqs->connection->write->active = 1; > + nqs->connection->write->ready = 1; > + > + if (!bidi) { if (bidi) ? > + nqs->connection->read->active = 1; > + } > + > return nqs->connection; This could've been simplified by adding a variable for nqs->connection, YMMV. > } > > @@ -534,6 +541,13 @@ ngx_quic_init_stream_handler(ngx_event_t > > ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic init stream"); > > + if ((qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0) { > + c->write->active = 1; > + c->write->ready = 1; > + } > + > + c->read->active = 1; > + > ngx_queue_remove(&qs->queue); > > c->listening->handler(c); > @@ -704,19 +718,6 @@ ngx_quic_create_stream(ngx_connection_t > > log->connection = sc->number; > > - if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0 > - || (id & NGX_QUIC_STREAM_SERVER_INITIATED)) > - { > - sc->write->active = 1; > - sc->write->ready = 1; > - } > - > - if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0 > - || (id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0) > - { > - sc->read->active = 1; > - } > - > if (id & NGX_QUIC_STREAM_UNIDIRECTIONAL) { > if (id & NGX_QUIC_STREAM_SERVER_INITIATED) { > qs->send_max_data = qc->ctp.initial_max_stream_data_uni; > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel ---- Roman Arutyunyan arut at nginx.com -------------- next part -------------- An HTML attachment was scrubbed... URL: From arut at nginx.com Tue Jan 17 11:23:58 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 17 Jan 2023 11:23:58 +0000 Subject: [PATCH 0 of 4] QUIC packet routing improvements In-Reply-To: References: Message-ID: Updated series. Patches 1 and 4 are removed. The remaining pacthes received minor updates. From arut at nginx.com Tue Jan 17 11:23:59 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 17 Jan 2023 11:23:59 +0000 Subject: [PATCH 1 of 4] QUIC: handle datagrams directly in ngx_quic_recvmsg() In-Reply-To: References: Message-ID: <7051cace147700a3e545.1673954639@ip-10-1-18-114.eu-central-1.compute.internal> # HG changeset patch # User Roman Arutyunyan # Date 1672752567 -14400 # Tue Jan 03 17:29:27 2023 +0400 # Branch quic # Node ID 7051cace147700a3e5456846dd4eadfdcadcd0c8 # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 QUIC: handle datagrams directly in ngx_quic_recvmsg(). Previously, ngx_quic_recvmsg() called client connection's read event handler to emulate normal event processing. Further, the read event handler handled the datagram by calling ngx_quic_handle_datagram(). Now ngx_quic_handle_datagram() is called directly from ngx_quic_recvmsg(), which simplifies the code. diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -17,8 +17,6 @@ static ngx_int_t ngx_quic_handle_statele static void ngx_quic_input_handler(ngx_event_t *rev); static void ngx_quic_close_handler(ngx_event_t *ev); -static ngx_int_t ngx_quic_handle_datagram(ngx_connection_t *c, ngx_buf_t *b, - ngx_quic_conf_t *conf); static ngx_int_t ngx_quic_handle_packet(ngx_connection_t *c, ngx_quic_conf_t *conf, ngx_quic_header_t *pkt); static ngx_int_t ngx_quic_handle_payload(ngx_connection_t *c, @@ -201,8 +199,7 @@ ngx_quic_apply_transport_params(ngx_conn void ngx_quic_run(ngx_connection_t *c, ngx_quic_conf_t *conf) { - ngx_int_t rc; - ngx_quic_connection_t *qc; + ngx_int_t rc; ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic run"); @@ -211,16 +208,6 @@ ngx_quic_run(ngx_connection_t *c, ngx_qu ngx_quic_close_connection(c, rc); return; } - - /* quic connection is now created */ - qc = ngx_quic_get_connection(c); - - ngx_add_timer(c->read, qc->tp.max_idle_timeout); - ngx_quic_connstate_dbg(c); - - c->read->handler = ngx_quic_input_handler; - - return; } @@ -340,6 +327,8 @@ ngx_quic_new_connection(ngx_connection_t c->idle = 1; ngx_reusable_connection(c, 1); + c->read->handler = ngx_quic_input_handler; + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic connection created"); @@ -397,8 +386,6 @@ ngx_quic_handle_stateless_reset(ngx_conn static void ngx_quic_input_handler(ngx_event_t *rev) { - ngx_int_t rc; - ngx_buf_t *b; ngx_connection_t *c; ngx_quic_connection_t *qc; @@ -432,29 +419,6 @@ ngx_quic_input_handler(ngx_event_t *rev) return; } - - b = c->udp->buffer; - if (b == NULL) { - return; - } - - rc = ngx_quic_handle_datagram(c, b, NULL); - - if (rc == NGX_ERROR) { - ngx_quic_close_connection(c, NGX_ERROR); - return; - } - - if (rc == NGX_DONE) { - return; - } - - /* rc == NGX_OK */ - - qc->send_timer_set = 0; - ngx_add_timer(rev, qc->tp.max_idle_timeout); - - ngx_quic_connstate_dbg(c); } @@ -654,7 +618,7 @@ ngx_quic_close_handler(ngx_event_t *ev) } -static ngx_int_t +ngx_int_t ngx_quic_handle_datagram(ngx_connection_t *c, ngx_buf_t *b, ngx_quic_conf_t *conf) { @@ -753,6 +717,11 @@ ngx_quic_handle_datagram(ngx_connection_ qc->error_reason = "QUIC flood detected"; return NGX_ERROR; } + + qc->send_timer_set = 0; + ngx_add_timer(c->read, qc->tp.max_idle_timeout); + + ngx_quic_connstate_dbg(c); } return NGX_OK; diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -260,6 +260,8 @@ struct ngx_quic_connection_s { }; +ngx_int_t ngx_quic_handle_datagram(ngx_connection_t *c, ngx_buf_t *b, + ngx_quic_conf_t *conf); ngx_int_t ngx_quic_apply_transport_params(ngx_connection_t *c, ngx_quic_tp_t *ctp); void ngx_quic_discard_ctx(ngx_connection_t *c, diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -264,12 +264,6 @@ ngx_quic_set_path(ngx_connection_t *c, n len = pkt->raw->last - pkt->raw->start; - if (c->udp->buffer == NULL) { - /* first ever packet in connection, path already exists */ - path = qc->path; - goto update; - } - probe = NULL; for (q = ngx_queue_head(&qc->paths); diff --git a/src/event/quic/ngx_event_quic_socket.c b/src/event/quic/ngx_event_quic_socket.c --- a/src/event/quic/ngx_event_quic_socket.c +++ b/src/event/quic/ngx_event_quic_socket.c @@ -61,6 +61,9 @@ ngx_quic_open_sockets(ngx_connection_t * } ngx_memcpy(qc->tp.initial_scid.data, qsock->sid.id, qsock->sid.len); + ngx_memcpy(&qsock->sockaddr.sockaddr, c->sockaddr, c->socklen); + qsock->socklen = c->socklen; + /* for all packets except first, this is set at udp layer */ c->udp = &qsock->udp; diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c --- a/src/event/quic/ngx_event_quic_udp.c +++ b/src/event/quic/ngx_event_quic_udp.c @@ -186,21 +186,11 @@ ngx_quic_recvmsg(ngx_event_t *ev) ngx_memcpy(&qsock->sockaddr.sockaddr, sockaddr, socklen); qsock->socklen = socklen; - c->udp->buffer = &buf; - - rev = c->read; - rev->ready = 1; - rev->active = 0; - - rev->handler(rev); - - if (c->udp) { - c->udp->buffer = NULL; + if (ngx_quic_handle_datagram(c, &buf, NULL) == NGX_ERROR) { + ngx_quic_close_connection(c, NGX_ERROR); + return; } - rev->ready = 0; - rev->active = 1; - goto next; } From arut at nginx.com Tue Jan 17 11:24:00 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 17 Jan 2023 11:24:00 +0000 Subject: [PATCH 2 of 4] QUIC: eliminated timeout handling in listen connection read event In-Reply-To: References: Message-ID: # HG changeset patch # User Roman Arutyunyan # Date 1670428974 0 # Wed Dec 07 16:02:54 2022 +0000 # Branch quic # Node ID cc74c21cddd87762c03c5ce5a9976b5f23d8823f # Parent 7051cace147700a3e5456846dd4eadfdcadcd0c8 QUIC: eliminated timeout handling in listen connection read event. The timeout is never set for QUIC. diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c --- a/src/event/quic/ngx_event_quic_udp.c +++ b/src/event/quic/ngx_event_quic_udp.c @@ -40,14 +40,6 @@ ngx_quic_recvmsg(ngx_event_t *ev) u_char msg_control[CMSG_SPACE(sizeof(ngx_addrinfo_t))]; #endif - if (ev->timedout) { - if (ngx_enable_accept_events((ngx_cycle_t *) ngx_cycle) != NGX_OK) { - return; - } - - ev->timedout = 0; - } - ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module); if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) { From arut at nginx.com Tue Jan 17 11:24:01 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 17 Jan 2023 11:24:01 +0000 Subject: [PATCH 3 of 4] QUIC: eBPF worker sockets In-Reply-To: References: Message-ID: <52681b76d4db0e9e814f.1673954641@ip-10-1-18-114.eu-central-1.compute.internal> # HG changeset patch # User Roman Arutyunyan # Date 1673867281 0 # Mon Jan 16 11:08:01 2023 +0000 # Branch quic # Node ID 52681b76d4db0e9e814f1e69b0d6b35f5ac630a2 # Parent cc74c21cddd87762c03c5ce5a9976b5f23d8823f QUIC: eBPF worker sockets. For each nginx worker, a worker socket is created. Worker sockets process existing QUIC connections bound to the worker, while listen sockets handle new connections. When shutting down a worker, listen sockets are closed as uaual, while worker sockets keep handling existing connections. Reuseport eBPF program looks up a worker socket by packet DCID and, if not found, chooses a listen socket based on packet address hash. The mode is enabled by "quic_bpf on" directive and is only available on Linux. Reuseport listen parameter is requried to enable the feature on a QUIC listen. diff --git a/auto/modules b/auto/modules --- a/auto/modules +++ b/auto/modules @@ -1363,11 +1363,11 @@ if [ $USE_OPENSSL_QUIC = YES ]; then . auto/module - if [ $QUIC_BPF = YES -a $SO_COOKIE_FOUND = YES ]; then + if [ $QUIC_BPF = YES ]; then ngx_module_type=CORE ngx_module_name=ngx_quic_bpf_module ngx_module_incs= - ngx_module_deps= + ngx_module_deps=src/event/quic/ngx_event_quic_bpf.h ngx_module_srcs="src/event/quic/ngx_event_quic_bpf.c \ src/event/quic/ngx_event_quic_bpf_code.c" ngx_module_libs= diff --git a/auto/options b/auto/options --- a/auto/options +++ b/auto/options @@ -171,8 +171,6 @@ USE_GEOIP=NO NGX_GOOGLE_PERFTOOLS=NO NGX_CPP_TEST=NO -SO_COOKIE_FOUND=NO - NGX_LIBATOMIC=NO NGX_CPU_CACHE_LINE= diff --git a/auto/os/linux b/auto/os/linux --- a/auto/os/linux +++ b/auto/os/linux @@ -234,7 +234,7 @@ ngx_include="sys/vfs.h"; . auto/incl # BPF sockhash -ngx_feature="BPF sockhash" +ngx_feature="BPF maps" ngx_feature_name="NGX_HAVE_BPF" ngx_feature_run=no ngx_feature_incs="#include @@ -245,7 +245,14 @@ ngx_feature_test="union bpf_attr attr = attr.map_flags = 0; attr.map_type = BPF_MAP_TYPE_SOCKHASH; + syscall(__NR_bpf, 0, &attr, 0); + attr.map_flags = 0; + attr.map_type = BPF_MAP_TYPE_SOCKMAP; + syscall(__NR_bpf, 0, &attr, 0); + + attr.map_flags = 0; + attr.map_type = BPF_MAP_TYPE_ARRAY; syscall(__NR_bpf, 0, &attr, 0);" . auto/feature @@ -259,23 +266,6 @@ if [ $ngx_found = yes ]; then fi -ngx_feature="SO_COOKIE" -ngx_feature_name="NGX_HAVE_SO_COOKIE" -ngx_feature_run=no -ngx_feature_incs="#include - #include " -ngx_feature_path= -ngx_feature_libs= -ngx_feature_test="socklen_t optlen = sizeof(uint64_t); - uint64_t cookie; - getsockopt(0, SOL_SOCKET, SO_COOKIE, &cookie, &optlen)" -. auto/feature - -if [ $ngx_found = yes ]; then - SO_COOKIE_FOUND=YES -fi - - # UDP segmentation offloading ngx_feature="UDP_SEGMENT" diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -1033,12 +1033,6 @@ ngx_close_listening_sockets(ngx_cycle_t ls = cycle->listening.elts; for (i = 0; i < cycle->listening.nelts; i++) { -#if (NGX_QUIC) - if (ls[i].quic) { - continue; - } -#endif - c = ls[i].connection; if (c) { diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h --- a/src/event/ngx_event.h +++ b/src/event/ngx_event.h @@ -73,6 +73,7 @@ struct ngx_event_s { /* to test on worker exit */ unsigned channel:1; unsigned resolver:1; + unsigned quic:1; unsigned cancelable:1; diff --git a/src/event/quic/bpf/makefile b/src/event/quic/bpf/makefile --- a/src/event/quic/bpf/makefile +++ b/src/event/quic/bpf/makefile @@ -1,4 +1,4 @@ -CFLAGS=-O2 -Wall +CFLAGS=-O2 -Wall $(MAKE_CFLAGS) LICENSE=BSD diff --git a/src/event/quic/bpf/ngx_quic_reuseport_helper.c b/src/event/quic/bpf/ngx_quic_reuseport_helper.c --- a/src/event/quic/bpf/ngx_quic_reuseport_helper.c +++ b/src/event/quic/bpf/ngx_quic_reuseport_helper.c @@ -44,97 +44,93 @@ char _license[] SEC("license") = LICENSE #define NGX_QUIC_SERVER_CID_LEN 20 -#define advance_data(nbytes) \ - offset += nbytes; \ - if (start + offset > end) { \ - debugmsg("cannot read %ld bytes at offset %ld", nbytes, offset); \ - goto failed; \ - } \ - data = start + offset - 1; - - -#define ngx_quic_parse_uint64(p) \ - (((__u64)(p)[0] << 56) | \ - ((__u64)(p)[1] << 48) | \ - ((__u64)(p)[2] << 40) | \ - ((__u64)(p)[3] << 32) | \ - ((__u64)(p)[4] << 24) | \ - ((__u64)(p)[5] << 16) | \ - ((__u64)(p)[6] << 8) | \ - ((__u64)(p)[7])) - -/* - * actual map object is created by the "bpf" system call, - * all pointers to this variable are replaced by the bpf loader - */ -struct bpf_map_def SEC("maps") ngx_quic_sockmap; +struct bpf_map_def SEC("maps") ngx_quic_listen; +struct bpf_map_def SEC("maps") ngx_quic_worker; +struct bpf_map_def SEC("maps") ngx_quic_nlisten; SEC(PROGNAME) -int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) +int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) \ { - int rc; - __u64 key; + int rc, flags; + __u32 zero, *nsockets, ns; size_t len, offset; - unsigned char *start, *end, *data, *dcid; + unsigned char *start, *end, dcid[NGX_QUIC_SERVER_CID_LEN]; start = ctx->data; - end = (unsigned char *) ctx->data_end; - offset = 0; + end = ctx->data_end; - advance_data(sizeof(struct udphdr)); /* data at UDP header */ - advance_data(1); /* data at QUIC flags */ - - if (data[0] & NGX_QUIC_PKT_LONG) { + offset = sizeof(struct udphdr) + 1; /* UDP header + QUIC flags */ + if (start + offset > end) { + goto bad_dgram; + } - advance_data(4); /* data at QUIC version */ - advance_data(1); /* data at DCID len */ + flags = start[offset - 1]; + if (flags & NGX_QUIC_PKT_LONG) { - len = data[0]; /* read DCID length */ - - if (len < 8) { - /* it's useless to search for key in such short DCID */ - return SK_PASS; + offset += 5; /* QUIC version + DCID len */ + if (start + offset > end) { + goto bad_dgram; } - } else { - len = NGX_QUIC_SERVER_CID_LEN; + len = start[offset - 1]; + if (len != NGX_QUIC_SERVER_CID_LEN) { + goto new_conn; + } + } + + if (start + offset + NGX_QUIC_SERVER_CID_LEN > end) { + goto bad_dgram; + } + + memcpy(dcid, start + offset, NGX_QUIC_SERVER_CID_LEN); + + rc = bpf_sk_select_reuseport(ctx, &ngx_quic_worker, dcid, 0); + + if (rc == 0) { + debugmsg("nginx quic socket selected by dcid"); + return SK_PASS; } - dcid = &data[1]; - advance_data(len); /* we expect the packet to have full DCID */ + if (rc != -ENOENT) { + debugmsg("nginx quic bpf_sk_select_reuseport() failed:%d", rc); + return SK_DROP; + } + +new_conn: - /* make verifier happy */ - if (dcid + sizeof(__u64) > end) { - goto failed; + zero = 0; + + nsockets = bpf_map_lookup_elem(&ngx_quic_nlisten, &zero); + + if (nsockets == NULL) { + debugmsg("nginx quic nsockets undefined"); + return SK_DROP; } - key = ngx_quic_parse_uint64(dcid); - - rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0); + ns = ctx->hash % *nsockets; - switch (rc) { - case 0: - debugmsg("nginx quic socket selected by key 0x%llx", key); - return SK_PASS; + rc = bpf_sk_select_reuseport(ctx, &ngx_quic_listen, &ns, 0); - /* kernel returns positive error numbers, errno.h defines positive */ - case -ENOENT: - debugmsg("nginx quic default route for key 0x%llx", key); - /* let the default reuseport logic decide which socket to choose */ + if (rc == 0) { + debugmsg("nginx quic socket selected by hash:%d", (int) ns); return SK_PASS; - - default: - debugmsg("nginx quic bpf_sk_select_reuseport err: %d key 0x%llx", - rc, key); - goto failed; } -failed: - /* - * SK_DROP will generate ICMP, but we may want to process "invalid" packet - * in userspace quic to investigate further and finally react properly - * (maybe ignore, maybe send something in response or close connection) - */ - return SK_PASS; + if (rc != -ENOENT) { + debugmsg("nginx quic bpf_sk_select_reuseport() failed:%d", rc); + return SK_DROP; + } + + (void) bpf_map_update_elem(&ngx_quic_nlisten, &zero, &ns, BPF_ANY); + + debugmsg("nginx quic cut sockets array:%d", (int) ns); + + return SK_DROP; + +bad_dgram: + + debugmsg("nginx quic bad datagram"); + + return SK_DROP; } diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -304,6 +304,10 @@ ngx_quic_new_connection(ngx_connection_t qc->congestion.ssthresh = (size_t) -1; qc->congestion.recovery_start = ngx_current_msec; + if (c->fd == c->listening->fd) { + qc->listen_bound = 1; + } + if (pkt->validated && pkt->retried) { qc->tp.retry_scid.len = pkt->dcid.len; qc->tp.retry_scid.data = ngx_pstrdup(c->pool, &pkt->dcid); @@ -413,6 +417,15 @@ ngx_quic_input_handler(ngx_event_t *rev) return; } + if (qc->listen_bound) { + c->fd = (ngx_socket_t) -1; + + qc->error = NGX_QUIC_ERR_NO_ERROR; + qc->error_reason = "graceful shutdown"; + ngx_quic_close_connection(c, NGX_ERROR); + return; + } + if (!qc->closing && qc->conf->shutdown) { qc->conf->shutdown(c); } @@ -885,14 +898,6 @@ ngx_quic_handle_packet(ngx_connection_t pkt->odcid = pkt->dcid; } - if (ngx_terminate || ngx_exiting) { - if (conf->retry) { - return ngx_quic_send_retry(c, conf, pkt); - } - - return NGX_ERROR; - } - c->log->action = "creating quic connection"; qc = ngx_quic_new_connection(c, conf, pkt); diff --git a/src/event/quic/ngx_event_quic_bpf.c b/src/event/quic/ngx_event_quic_bpf.c --- a/src/event/quic/ngx_event_quic_bpf.c +++ b/src/event/quic/ngx_event_quic_bpf.c @@ -6,6 +6,8 @@ #include #include +#include +#include #define NGX_QUIC_BPF_VARNAME "NGINX_BPF_MAPS" @@ -26,39 +28,57 @@ typedef struct { ngx_queue_t queue; - int map_fd; + + int listen_map; + int worker_map; + int nlisten_map; struct sockaddr *sockaddr; socklen_t socklen; - ngx_uint_t unused; /* unsigned unused:1; */ -} ngx_quic_sock_group_t; + + ngx_array_t listening; + + ngx_uint_t nlisten; + ngx_uint_t old_nlisten; +} ngx_quic_bpf_group_t; + + +typedef struct { + ngx_socket_t fd; + ngx_listening_t *listening; + ngx_connection_t *connection; +} ngx_quic_bpf_listening_t; typedef struct { ngx_flag_t enabled; - ngx_uint_t map_size; - ngx_queue_t groups; /* of ngx_quic_sock_group_t */ + ngx_uint_t max_connection_ids; + ngx_uint_t max_workers; + ngx_queue_t groups; } ngx_quic_bpf_conf_t; static void *ngx_quic_bpf_create_conf(ngx_cycle_t *cycle); +static char *ngx_quic_bpf_init_conf(ngx_cycle_t *cycle, void *conf); static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle); static void ngx_quic_bpf_cleanup(void *data); static ngx_inline void ngx_quic_bpf_close(ngx_log_t *log, int fd, const char *name); -static ngx_quic_sock_group_t *ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, +static ngx_quic_bpf_group_t *ngx_quic_bpf_find_group(ngx_cycle_t *cycle, + ngx_listening_t *ls); +static ngx_quic_bpf_group_t *ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, ngx_listening_t *ls); -static ngx_quic_sock_group_t *ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, - struct sockaddr *sa, socklen_t socklen); -static ngx_quic_sock_group_t *ngx_quic_bpf_create_group(ngx_cycle_t *cycle, +static ngx_quic_bpf_group_t *ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls); -static ngx_quic_sock_group_t *ngx_quic_bpf_get_group(ngx_cycle_t *cycle, +static ngx_int_t ngx_quic_bpf_inherit_fd(ngx_cycle_t *cycle, int fd); +static ngx_quic_bpf_group_t *ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls); static ngx_int_t ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle, ngx_listening_t *ls); -static uint64_t ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log); +static ngx_int_t ngx_quic_bpf_add_worker_socket(ngx_cycle_t *cycle, + ngx_quic_bpf_group_t *grp, ngx_listening_t *ls); static ngx_int_t ngx_quic_bpf_export_maps(ngx_cycle_t *cycle); static ngx_int_t ngx_quic_bpf_import_maps(ngx_cycle_t *cycle); @@ -82,7 +102,7 @@ static ngx_command_t ngx_quic_bpf_comma static ngx_core_module_t ngx_quic_bpf_module_ctx = { ngx_string("quic_bpf"), ngx_quic_bpf_create_conf, - NULL + ngx_quic_bpf_init_conf }; @@ -113,7 +133,6 @@ ngx_quic_bpf_create_conf(ngx_cycle_t *cy } bcf->enabled = NGX_CONF_UNSET; - bcf->map_size = NGX_CONF_UNSET_UINT; ngx_queue_init(&bcf->groups); @@ -121,12 +140,41 @@ ngx_quic_bpf_create_conf(ngx_cycle_t *cy } +static char * +ngx_quic_bpf_init_conf(ngx_cycle_t *cycle, void *conf) +{ + ngx_quic_bpf_conf_t *bcf = conf; + + ngx_quic_bpf_conf_t *obcf; + + ngx_conf_init_value(bcf->enabled, 0); + + if (cycle->old_cycle->conf_ctx == NULL) { + return NGX_CONF_OK; + } + + obcf = ngx_quic_bpf_get_conf(cycle->old_cycle); + if (obcf == NULL) { + return NGX_CONF_OK; + } + + if (obcf->enabled != bcf->enabled) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, + "cannot change \"quic_bpf\" after reload, ignoring"); + bcf->enabled = obcf->enabled; + } + + return NGX_CONF_OK; +} + + static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle) { ngx_uint_t i; ngx_listening_t *ls; ngx_core_conf_t *ccf; + ngx_event_conf_t *ecf; ngx_pool_cleanup_t *cln; ngx_quic_bpf_conf_t *bcf; @@ -138,12 +186,16 @@ ngx_quic_bpf_module_init(ngx_cycle_t *cy return NGX_OK; } - ccf = ngx_core_get_conf(cycle); bcf = ngx_quic_bpf_get_conf(cycle); + if (!bcf->enabled) { + return NGX_OK; + } - ngx_conf_init_value(bcf->enabled, 0); + ccf = ngx_core_get_conf(cycle); + ecf = ngx_event_get_conf(cycle->conf_ctx, ngx_event_core_module); - bcf->map_size = ccf->worker_processes * 4; + bcf->max_connection_ids = ecf->connections * NGX_QUIC_MAX_SERVER_IDS; + bcf->max_workers = ccf->worker_processes * 4; cln = ngx_pool_cleanup_add(cycle->pool, 0); if (cln == NULL) { @@ -153,6 +205,8 @@ ngx_quic_bpf_module_init(ngx_cycle_t *cy cln->data = bcf; cln->handler = ngx_quic_bpf_cleanup; + ls = cycle->listening.elts; + if (ngx_inherited && ngx_is_init_cycle(cycle->old_cycle)) { if (ngx_quic_bpf_import_maps(cycle) != NGX_OK) { goto failed; @@ -208,16 +262,32 @@ ngx_quic_bpf_cleanup(void *data) { ngx_quic_bpf_conf_t *bcf = (ngx_quic_bpf_conf_t *) data; - ngx_queue_t *q; - ngx_quic_sock_group_t *grp; + ngx_uint_t i; + ngx_queue_t *q; + ngx_quic_bpf_group_t *grp; + ngx_quic_bpf_listening_t *bls; for (q = ngx_queue_head(&bcf->groups); q != ngx_queue_sentinel(&bcf->groups); q = ngx_queue_next(q)) { - grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); + grp = ngx_queue_data(q, ngx_quic_bpf_group_t, queue); + + ngx_quic_bpf_close(ngx_cycle->log, grp->listen_map, "listen"); + ngx_quic_bpf_close(ngx_cycle->log, grp->worker_map, "worker"); + ngx_quic_bpf_close(ngx_cycle->log, grp->nlisten_map, "nlisten"); + + bls = grp->listening.elts; - ngx_quic_bpf_close(ngx_cycle->log, grp->map_fd, "map"); + for (i = 0; i < grp->listening.nelts; i++) { + if (bls[i].fd != (ngx_socket_t) -1) { + if (ngx_close_socket(bls[i].fd) == -1) { + ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, + ngx_socket_errno, + ngx_close_socket_n " failed"); + } + } + } } } @@ -230,25 +300,32 @@ ngx_quic_bpf_close(ngx_log_t *log, int f } ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, - "quic bpf close %s fd:%d failed", name, fd); + "QUIC BPF close %s map fd:%d failed", name, fd); } -static ngx_quic_sock_group_t * -ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, ngx_listening_t *ls) +static ngx_quic_bpf_group_t * +ngx_quic_bpf_find_group(ngx_cycle_t *cycle, ngx_listening_t *ls) { - ngx_queue_t *q; - ngx_quic_sock_group_t *grp; + ngx_queue_t *q; + ngx_quic_bpf_conf_t *bcf; + ngx_quic_bpf_group_t *grp; + + bcf = ngx_quic_bpf_get_conf(cycle); + + if (!bcf->enabled || !ls->quic || !ls->reuseport) { + return NULL; + } for (q = ngx_queue_head(&bcf->groups); q != ngx_queue_sentinel(&bcf->groups); q = ngx_queue_next(q)) { - grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); + grp = ngx_queue_data(q, ngx_quic_bpf_group_t, queue); if (ngx_cmp_sockaddr(ls->sockaddr, ls->socklen, grp->sockaddr, grp->socklen, 1) - == NGX_OK) + == 0) { return grp; } @@ -258,26 +335,32 @@ ngx_quic_bpf_find_group(ngx_quic_bpf_con } -static ngx_quic_sock_group_t * -ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, struct sockaddr *sa, - socklen_t socklen) +static ngx_quic_bpf_group_t * +ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, ngx_listening_t *ls) { ngx_quic_bpf_conf_t *bcf; - ngx_quic_sock_group_t *grp; + ngx_quic_bpf_group_t *grp; bcf = ngx_quic_bpf_get_conf(cycle); - grp = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_sock_group_t)); + grp = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_bpf_group_t)); if (grp == NULL) { return NULL; } - grp->socklen = socklen; - grp->sockaddr = ngx_palloc(cycle->pool, socklen); - if (grp->sockaddr == NULL) { + grp->listen_map = -1; + grp->worker_map = -1; + grp->nlisten_map = -1; + + grp->sockaddr = ls->sockaddr; + grp->socklen = ls->socklen; + + if (ngx_array_init(&grp->listening, cycle->pool, 1, + sizeof(ngx_quic_bpf_listening_t)) + != NGX_OK) + { return NULL; } - ngx_memcpy(grp->sockaddr, sa, socklen); ngx_queue_insert_tail(&bcf->groups, &grp->queue); @@ -285,50 +368,72 @@ ngx_quic_bpf_alloc_group(ngx_cycle_t *cy } -static ngx_quic_sock_group_t * +static ngx_quic_bpf_group_t * ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls) { - int progfd, failed, flags, rc; - ngx_quic_bpf_conf_t *bcf; - ngx_quic_sock_group_t *grp; + int progfd, failed; + ngx_quic_bpf_conf_t *bcf; + ngx_quic_bpf_group_t *grp; bcf = ngx_quic_bpf_get_conf(cycle); - if (!bcf->enabled) { - return NULL; - } - - grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen); + grp = ngx_quic_bpf_alloc_group(cycle, ls); if (grp == NULL) { return NULL; } - grp->map_fd = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKHASH, - sizeof(uint64_t), sizeof(uint64_t), - bcf->map_size, 0); - if (grp->map_fd == -1) { + grp->listen_map = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKMAP, + sizeof(uint32_t), sizeof(uint64_t), + bcf->max_workers, 0); + if (grp->listen_map == -1) { goto failed; } - flags = fcntl(grp->map_fd, F_GETFD); - if (flags == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, errno, - "quic bpf getfd failed"); - goto failed; - } - - /* need to inherit map during binary upgrade after exec */ - flags &= ~FD_CLOEXEC; - - rc = fcntl(grp->map_fd, F_SETFD, flags); - if (rc == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, errno, - "quic bpf setfd failed"); + if (ngx_quic_bpf_inherit_fd(cycle, grp->listen_map) != NGX_OK) { goto failed; } ngx_bpf_program_link(&ngx_quic_reuseport_helper, - "ngx_quic_sockmap", grp->map_fd); + "ngx_quic_listen", grp->listen_map); + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "quic bpf listen map created fd:%d", grp->listen_map); + + + grp->worker_map = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKHASH, + NGX_QUIC_SERVER_CID_LEN, sizeof(uint64_t), + bcf->max_connection_ids, 0); + if (grp->worker_map == -1) { + goto failed; + } + + if (ngx_quic_bpf_inherit_fd(cycle, grp->worker_map) != NGX_OK) { + goto failed; + } + + ngx_bpf_program_link(&ngx_quic_reuseport_helper, + "ngx_quic_worker", grp->worker_map); + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "quic bpf worker map created fd:%d", grp->worker_map); + + + grp->nlisten_map = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_ARRAY, + sizeof(uint32_t), sizeof(uint32_t), 1, 0); + if (grp->nlisten_map == -1) { + goto failed; + } + + if (ngx_quic_bpf_inherit_fd(cycle, grp->nlisten_map) != NGX_OK) { + goto failed; + } + + ngx_bpf_program_link(&ngx_quic_reuseport_helper, + "ngx_quic_nlisten", grp->nlisten_map); + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "quic bpf nlisten map created fd:%d", grp->nlisten_map); + progfd = ngx_bpf_load_program(cycle->log, &ngx_quic_reuseport_helper); if (progfd < 0) { @@ -352,14 +457,116 @@ ngx_quic_bpf_create_group(ngx_cycle_t *c goto failed; } - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, - "quic bpf sockmap created fd:%d", grp->map_fd); return grp; failed: - if (grp->map_fd != -1) { - ngx_quic_bpf_close(cycle->log, grp->map_fd, "map"); + if (grp->listen_map != -1) { + ngx_quic_bpf_close(cycle->log, grp->listen_map, "listen"); + } + + if (grp->worker_map != -1) { + ngx_quic_bpf_close(cycle->log, grp->worker_map, "worker"); + } + + if (grp->nlisten_map != -1) { + ngx_quic_bpf_close(cycle->log, grp->nlisten_map, "nlisten"); + } + + ngx_queue_remove(&grp->queue); + + return NULL; +} + + +static ngx_int_t +ngx_quic_bpf_inherit_fd(ngx_cycle_t *cycle, int fd) +{ + int flags; + + flags = fcntl(fd, F_GETFD); + if (flags == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "fcntl(F_GETFD) failed"); + return NGX_ERROR; + } + + flags &= ~FD_CLOEXEC; + + if (fcntl(fd, F_SETFD, flags) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "fcntl(F_SETFD) failed"); + return NGX_ERROR; + } + + return NGX_OK; +} + + +static ngx_quic_bpf_group_t * +ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls) +{ + ngx_quic_bpf_conf_t *old_bcf; + ngx_quic_bpf_group_t *grp, *ogrp; + + grp = ngx_quic_bpf_find_group(cycle, ls); + if (grp) { + return grp; + } + + old_bcf = ngx_quic_bpf_get_old_conf(cycle); + if (old_bcf == NULL) { + return ngx_quic_bpf_create_group(cycle, ls); + } + + ogrp = ngx_quic_bpf_find_group(cycle->old_cycle, ls); + if (ogrp == NULL) { + return ngx_quic_bpf_create_group(cycle, ls); + } + + grp = ngx_quic_bpf_alloc_group(cycle, ls); + if (grp == NULL) { + return NULL; + } + + grp->old_nlisten = ogrp->nlisten; + + grp->listen_map = dup(ogrp->listen_map); + if (grp->listen_map == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "failed to duplicate QUIC BPF listen map"); + + goto failed; + } + + grp->worker_map = dup(ogrp->worker_map); + if (grp->worker_map == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "failed to duplicate QUIC BPF worker map"); + goto failed; + } + + grp->nlisten_map = dup(ogrp->nlisten_map); + if (grp->nlisten_map == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "failed to duplicate QUIC BPF nlisten map"); + goto failed; + } + + return grp; + +failed: + + if (grp->listen_map != -1) { + ngx_quic_bpf_close(cycle->log, grp->listen_map, "listen"); + } + + if (grp->worker_map != -1) { + ngx_quic_bpf_close(cycle->log, grp->worker_map, "worker"); + } + + if (grp->nlisten_map != -1) { + ngx_quic_bpf_close(cycle->log, grp->nlisten_map, "nlisten"); } ngx_queue_remove(&grp->queue); @@ -368,129 +575,173 @@ failed: } -static ngx_quic_sock_group_t * -ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls) -{ - ngx_quic_bpf_conf_t *bcf, *old_bcf; - ngx_quic_sock_group_t *grp, *ogrp; - - bcf = ngx_quic_bpf_get_conf(cycle); - - grp = ngx_quic_bpf_find_group(bcf, ls); - if (grp) { - return grp; - } - - old_bcf = ngx_quic_bpf_get_old_conf(cycle); - - if (old_bcf == NULL) { - return ngx_quic_bpf_create_group(cycle, ls); - } - - ogrp = ngx_quic_bpf_find_group(old_bcf, ls); - if (ogrp == NULL) { - return ngx_quic_bpf_create_group(cycle, ls); - } - - grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen); - if (grp == NULL) { - return NULL; - } - - grp->map_fd = dup(ogrp->map_fd); - if (grp->map_fd == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, - "quic bpf failed to duplicate bpf map descriptor"); - - ngx_queue_remove(&grp->queue); - - return NULL; - } - - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, - "quic bpf sockmap fd duplicated old:%d new:%d", - ogrp->map_fd, grp->map_fd); - - return grp; -} - - static ngx_int_t ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle, ngx_listening_t *ls) { - uint64_t cookie; - ngx_quic_bpf_conf_t *bcf; - ngx_quic_sock_group_t *grp; - - bcf = ngx_quic_bpf_get_conf(cycle); + uint32_t zero, key; + ngx_quic_bpf_group_t *grp; grp = ngx_quic_bpf_get_group(cycle, ls); + if (grp == NULL) { + return NGX_ERROR; + } - if (grp == NULL) { - if (!bcf->enabled) { - return NGX_OK; - } - + if (ngx_quic_bpf_add_worker_socket(cycle, grp, ls) != NGX_OK) { return NGX_ERROR; } - grp->unused = 0; + key = ls->worker; - cookie = ngx_quic_bpf_socket_key(ls->fd, cycle->log); - if (cookie == (uint64_t) NGX_ERROR) { + if (ngx_bpf_map_update(grp->listen_map, &key, &ls->fd, BPF_ANY) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "failed to update QUIC BPF listen map"); return NGX_ERROR; } - /* map[cookie] = socket; for use in kernel helper */ - if (ngx_bpf_map_update(grp->map_fd, &cookie, &ls->fd, BPF_ANY) == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, - "quic bpf failed to update socket map key=%xL", cookie); - return NGX_ERROR; + if (grp->nlisten >= ls->worker + 1) { + return NGX_OK; } - ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0, - "quic bpf sockmap fd:%d add socket:%d cookie:0x%xL worker:%ui", - grp->map_fd, ls->fd, cookie, ls->worker); + grp->nlisten = ls->worker + 1; + + if (grp->nlisten <= grp->old_nlisten) { + return NGX_OK; + } - /* do not inherit this socket */ - ls->ignore = 1; + zero = 0; + key = grp->nlisten; + + if (ngx_bpf_map_update(grp->nlisten_map, &zero, &key, BPF_ANY) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "failed to update QUIC BPF nlisten map"); + return NGX_ERROR; + } return NGX_OK; } -static uint64_t -ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log) +static ngx_int_t +ngx_quic_bpf_add_worker_socket(ngx_cycle_t *cycle, ngx_quic_bpf_group_t *grp, + ngx_listening_t *ls) { - uint64_t cookie; - socklen_t optlen; + int value; + ngx_uint_t i, n; + ngx_socket_t s; + ngx_quic_bpf_listening_t *bls; + + s = ngx_socket(ls->sockaddr->sa_family, SOCK_DGRAM, 0); + if (s == (ngx_socket_t) -1) { + ngx_log_error(NGX_LOG_ERR, cycle->log, ngx_socket_errno, + ngx_socket_n " failed"); + return NGX_ERROR; + } - optlen = sizeof(cookie); + if (ngx_nonblocking(s) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + ngx_nonblocking_n " worker socket failed"); + goto failed; + } + + value = 1; - if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) { - ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, - "quic bpf getsockopt(SO_COOKIE) failed"); + if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "setsockopt(SO_REUSEADDR) worker socket failed"); + goto failed; + } - return (ngx_uint_t) NGX_ERROR; + if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "setsockopt(SO_REUSEPORT) worker socket failed"); + goto failed; } - return cookie; +#if (NGX_HAVE_IP_PKTINFO) + if (ls->wildcard && ls->sockaddr->sa_family == AF_INET) { + if (setsockopt(s, IPPROTO_IP, IP_PKTINFO, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "setsockopt(IP_PKTINFO) worker socket failed"); + goto failed; + } + } +#endif + +#if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO) + if (ls->wildcard && ls->sockaddr->sa_family == AF_INET6) { + if (setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "setsockopt(IPV6_RECVPKTINFO) worker socket failed"); + } + } +#endif + + if (bind(s, ls->sockaddr, ls->socklen) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "bind() failed"); + goto failed; + } + + if (ls->worker >= grp->listening.nelts) { + n = ls->worker + 1 - grp->listening.nelts; + + bls = ngx_array_push_n(&grp->listening, n); + if (bls == NULL) { + goto failed; + } + + ngx_memzero(bls, n * sizeof(ngx_quic_bpf_listening_t)); + + for (i = 0; i < n; i++) { + bls[i].fd = (ngx_socket_t) -1; + } + } + + bls = grp->listening.elts; + bls[ls->worker].fd = s; + bls[ls->worker].listening = ls; + + return NGX_OK; + +failed: + + if (ngx_close_socket(s) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + ngx_close_socket_n " failed"); + } + + return NGX_ERROR; } - static ngx_int_t ngx_quic_bpf_export_maps(ngx_cycle_t *cycle) { - u_char *p, *buf; - size_t len; - ngx_str_t *var; - ngx_queue_t *q; - ngx_core_conf_t *ccf; - ngx_quic_bpf_conf_t *bcf; - ngx_quic_sock_group_t *grp; + u_char *p, *buf; + size_t len; + ngx_str_t *var; + ngx_queue_t *q; + ngx_core_conf_t *ccf; + ngx_quic_bpf_conf_t *bcf; + ngx_quic_bpf_group_t *grp; + + bcf = ngx_quic_bpf_get_conf(cycle); + if (!bcf->enabled) { + return NGX_OK; + } ccf = ngx_core_get_conf(cycle); - bcf = ngx_quic_bpf_get_conf(cycle); len = sizeof(NGX_QUIC_BPF_VARNAME) + 1; @@ -498,24 +749,26 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cy while (q != ngx_queue_sentinel(&bcf->groups)) { - grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); + grp = ngx_queue_data(q, ngx_quic_bpf_group_t, queue); q = ngx_queue_next(q); - if (grp->unused) { + if (grp->nlisten == 0) { /* * map was inherited, but it is not used in this configuration; * do not pass such map further and drop the group to prevent * interference with changes during reload */ - ngx_quic_bpf_close(cycle->log, grp->map_fd, "map"); + ngx_quic_bpf_close(cycle->log, grp->listen_map, "listen"); + ngx_quic_bpf_close(cycle->log, grp->worker_map, "worker"); + ngx_quic_bpf_close(cycle->log, grp->nlisten_map, "nlisten"); + ngx_queue_remove(&grp->queue); - continue; } - len += NGX_INT32_LEN + 1 + NGX_SOCKADDR_STRLEN + 1; + len += (NGX_INT32_LEN + 1) * 3 + NGX_SOCKADDR_STRLEN + 1; } len++; @@ -525,22 +778,23 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cy return NGX_ERROR; } - p = ngx_cpymem(buf, NGX_QUIC_BPF_VARNAME "=", - sizeof(NGX_QUIC_BPF_VARNAME)); + p = ngx_cpymem(buf, NGX_QUIC_BPF_VARNAME "=", sizeof(NGX_QUIC_BPF_VARNAME)); for (q = ngx_queue_head(&bcf->groups); q != ngx_queue_sentinel(&bcf->groups); q = ngx_queue_next(q)) { - grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); + grp = ngx_queue_data(q, ngx_quic_bpf_group_t, queue); - p = ngx_sprintf(p, "%ud", grp->map_fd); - + p = ngx_sprintf(p, "%ud", grp->listen_map); + *p++ = NGX_QUIC_BPF_ADDRSEP; + p = ngx_sprintf(p, "%ud", grp->worker_map); + *p++ = NGX_QUIC_BPF_ADDRSEP; + p = ngx_sprintf(p, "%ud", grp->nlisten_map); *p++ = NGX_QUIC_BPF_ADDRSEP; p += ngx_sock_ntop(grp->sockaddr, grp->socklen, p, NGX_SOCKADDR_STRLEN, 1); - *p++ = NGX_QUIC_BPF_VARSEP; } @@ -561,12 +815,14 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cy static ngx_int_t ngx_quic_bpf_import_maps(ngx_cycle_t *cycle) { - int s; - u_char *inherited, *p, *v; - ngx_uint_t in_fd; - ngx_addr_t tmp; - ngx_quic_bpf_conf_t *bcf; - ngx_quic_sock_group_t *grp; + int fds[3]; + u_char *inherited, *p, *v; + uint32_t zero, nlisten; + ngx_int_t fd; + ngx_uint_t nfd; + ngx_addr_t tmp; + ngx_quic_bpf_conf_t *bcf; + ngx_quic_bpf_group_t *grp; inherited = (u_char *) getenv(NGX_QUIC_BPF_VARNAME); @@ -574,13 +830,13 @@ ngx_quic_bpf_import_maps(ngx_cycle_t *cy return NGX_OK; } + ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, + "using inherited QUIC BPF maps from \"%s\"", inherited); + bcf = ngx_quic_bpf_get_conf(cycle); -#if (NGX_SUPPRESS_WARN) - s = -1; -#endif - - in_fd = 1; + zero = 0; + nfd = 0; for (p = inherited, v = p; *p; p++) { @@ -588,63 +844,69 @@ ngx_quic_bpf_import_maps(ngx_cycle_t *cy case NGX_QUIC_BPF_ADDRSEP: - if (!in_fd) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited env"); - return NGX_ERROR; - } - in_fd = 0; - - s = ngx_atoi(v, p - v); - if (s == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited map fd"); - return NGX_ERROR; + if (nfd > 2) { + goto failed; } + fd = ngx_atoi(v, p - v); + if (fd == NGX_ERROR) { + goto failed; + } + + fds[nfd++] = fd; v = p + 1; break; case NGX_QUIC_BPF_VARSEP: - if (in_fd) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited env"); - return NGX_ERROR; + if (nfd != 3) { + goto failed; } - in_fd = 1; - grp = ngx_pcalloc(cycle->pool, - sizeof(ngx_quic_sock_group_t)); + grp = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_bpf_group_t)); if (grp == NULL) { return NGX_ERROR; } - grp->map_fd = s; - - if (ngx_parse_addr_port(cycle->pool, &tmp, v, p - v) + if (ngx_array_init(&grp->listening, cycle->pool, 1, + sizeof(ngx_quic_bpf_listening_t)) != NGX_OK) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited" - " address '%*s'", p - v , v); + return NGX_ERROR; + } + + grp->listen_map = fds[0]; + grp->worker_map = fds[1]; + grp->nlisten_map = fds[2]; - ngx_quic_bpf_close(cycle->log, s, "inherited map"); + if (ngx_parse_addr_port(cycle->pool, &tmp, v, p - v) != NGX_OK) { + goto failed; + } + grp->sockaddr = ngx_pcalloc(cycle->pool, tmp.socklen); + if (grp->sockaddr == NULL) { return NGX_ERROR; } - grp->sockaddr = tmp.sockaddr; + ngx_memcpy(grp->sockaddr, tmp.sockaddr, tmp.socklen); grp->socklen = tmp.socklen; - grp->unused = 1; + if (ngx_bpf_map_lookup(fds[2], &zero, &nlisten) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "failed to lookup QUIC BPF listen number"); + return NGX_ERROR; + } + + grp->old_nlisten = nlisten; ngx_queue_insert_tail(&bcf->groups, &grp->queue); - ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + ngx_log_debug5(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "quic bpf sockmap inherited with " - "fd:%d address:%*s", - grp->map_fd, p - v, v); + "fds:%d/%d/%d address:%*s", + fds[0], fds[1], fds[2], p - v, v); + + nfd = 0; v = p + 1; break; @@ -654,4 +916,127 @@ ngx_quic_bpf_import_maps(ngx_cycle_t *cy } return NGX_OK; + +failed: + + ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, + "failed to parse inherited QUIC BPF variable"); + + return NGX_ERROR; } + + +ngx_int_t +ngx_quic_bpf_get_client_connection(ngx_connection_t *lc, ngx_connection_t **pc) +{ + ngx_event_t *rev; + ngx_connection_t *c; + ngx_quic_bpf_group_t *grp; + ngx_quic_bpf_listening_t *bpf_listening, *bls; + + grp = ngx_quic_bpf_find_group((ngx_cycle_t *) ngx_cycle, lc->listening); + + if (grp == NULL || ngx_worker >= grp->listening.nelts) { + return NGX_OK; + } + + bpf_listening = grp->listening.elts; + bls = &bpf_listening[ngx_worker]; + + if (bls->fd == (ngx_socket_t) -1) { + return NGX_OK; + } + + if (bls->connection == NULL) { + c = ngx_get_connection(bls->fd, lc->log); + if (c == NULL) { + return NGX_ERROR; + } + + c->type = SOCK_DGRAM; + c->log = lc->log; + c->listening = bls->listening; + + rev = c->read; + rev->quic = 1; + rev->log = c->log; + rev->handler = ngx_quic_recvmsg; + + if (ngx_add_event(rev, NGX_READ_EVENT, 0) == NGX_ERROR) { + ngx_free_connection(c); + return NGX_ERROR; + } + + bls->connection = c; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, lc->log, 0, + "quic bpf worker socket connection fd:%d", bls->fd); + + } + + *pc = ngx_get_connection(bls->fd, lc->log); + if (*pc == NULL) { + return NGX_ERROR; + } + + (*pc)->shared = 1; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, lc->log, 0, + "quic bpf client connection fd:%d", bls->fd); + + return NGX_OK; +} + + +ngx_int_t +ngx_quic_bpf_insert(ngx_connection_t *c, ngx_quic_connection_t *qc, + ngx_quic_socket_t *qsock) +{ + ngx_quic_bpf_group_t *grp; + + if (qsock->sid.len != NGX_QUIC_SERVER_CID_LEN) { + /* route by address */ + return NGX_OK; + } + + grp = ngx_quic_bpf_find_group((ngx_cycle_t *) ngx_cycle, c->listening); + if (grp == NULL) { + return NGX_OK; + } + + if (ngx_bpf_map_update(grp->worker_map, qsock->sid.id, &c->fd, BPF_ANY) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno, + "failed to update QUIC BPF worker map"); + return NGX_ERROR; + } + + return NGX_OK; +} + + +ngx_int_t +ngx_quic_bpf_delete(ngx_connection_t *c, ngx_quic_connection_t *qc, + ngx_quic_socket_t *qsock) +{ + ngx_quic_bpf_group_t *grp; + + if (qsock->sid.len != NGX_QUIC_SERVER_CID_LEN) { + /* route by address */ + return NGX_OK; + } + + grp = ngx_quic_bpf_find_group((ngx_cycle_t *) ngx_cycle, c->listening); + if (grp == NULL) { + return NGX_OK; + } + + if (ngx_bpf_map_delete(grp->worker_map, qsock->sid.id) == -1) { + ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno, + "failed to update QUIC BPF worker map"); + return NGX_ERROR; + } + + return NGX_OK; +} diff --git a/src/event/quic/ngx_event_quic_bpf.h b/src/event/quic/ngx_event_quic_bpf.h new file mode 100644 --- /dev/null +++ b/src/event/quic/ngx_event_quic_bpf.h @@ -0,0 +1,23 @@ + +/* + * Copyright (C) Nginx, Inc. + */ + + +#ifndef _NGX_EVENT_QUIC_BPF_H_INCLUDED_ +#define _NGX_EVENT_QUIC_BPF_H_INCLUDED_ + + +#include +#include + + +ngx_int_t ngx_quic_bpf_get_client_connection(ngx_connection_t *lc, + ngx_connection_t **pc); +ngx_int_t ngx_quic_bpf_insert(ngx_connection_t *c, ngx_quic_connection_t *qc, + ngx_quic_socket_t *qsock); +ngx_int_t ngx_quic_bpf_delete(ngx_connection_t *c, ngx_quic_connection_t *qc, + ngx_quic_socket_t *qsock); + + +#endif /* _NGX_EVENT_QUIC_BPF_H_INCLUDED_ */ diff --git a/src/event/quic/ngx_event_quic_bpf_code.c b/src/event/quic/ngx_event_quic_bpf_code.c --- a/src/event/quic/ngx_event_quic_bpf_code.c +++ b/src/event/quic/ngx_event_quic_bpf_code.c @@ -7,71 +7,146 @@ static ngx_bpf_reloc_t bpf_reloc_prog_ngx_quic_reuseport_helper[] = { - { "ngx_quic_sockmap", 55 }, + { "ngx_quic_worker", 82 }, + { "ngx_quic_nlisten", 99 }, + { "ngx_quic_listen", 110 }, + { "ngx_quic_nlisten", 127 }, }; static struct bpf_insn bpf_insn_prog_ngx_quic_reuseport_helper[] = { /* opcode dst src offset imm */ - { 0x79, BPF_REG_4, BPF_REG_1, (int16_t) 0, 0x0 }, - { 0x79, BPF_REG_3, BPF_REG_1, (int16_t) 8, 0x0 }, - { 0xbf, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 }, - { 0x2d, BPF_REG_2, BPF_REG_3, (int16_t) 54, 0x0 }, - { 0xbf, BPF_REG_5, BPF_REG_4, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x9 }, - { 0x2d, BPF_REG_5, BPF_REG_3, (int16_t) 51, 0x0 }, - { 0xb7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x14 }, - { 0xb7, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x9 }, - { 0x71, BPF_REG_6, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x67, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x38 }, - { 0xc7, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x38 }, - { 0x65, BPF_REG_6, BPF_REG_0, (int16_t) 10, 0xffffffff }, - { 0xbf, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xd }, - { 0x2d, BPF_REG_2, BPF_REG_3, (int16_t) 42, 0x0 }, - { 0xbf, BPF_REG_5, BPF_REG_4, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0xe }, - { 0x2d, BPF_REG_5, BPF_REG_3, (int16_t) 39, 0x0 }, - { 0xb7, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xe }, - { 0x71, BPF_REG_5, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0xb7, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x8 }, - { 0x2d, BPF_REG_6, BPF_REG_5, (int16_t) 35, 0x0 }, - { 0xf, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x0 }, - { 0xf, BPF_REG_4, BPF_REG_5, (int16_t) 0, 0x0 }, - { 0x2d, BPF_REG_4, BPF_REG_3, (int16_t) 32, 0x0 }, - { 0xbf, BPF_REG_4, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x9 }, - { 0x2d, BPF_REG_4, BPF_REG_3, (int16_t) 29, 0x0 }, - { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 1, 0x0 }, + { 0xbf, BPF_REG_6, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x79, BPF_REG_2, BPF_REG_6, (int16_t) 8, 0x0 }, + { 0x79, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x9 }, + { 0x2d, BPF_REG_3, BPF_REG_2, (int16_t) 124, 0x0 }, + { 0xb7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x9 }, + { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 8, 0x0 }, { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x38 }, - { 0x71, BPF_REG_3, BPF_REG_2, (int16_t) 2, 0x0 }, - { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x30 }, - { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 3, 0x0 }, - { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x28 }, - { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 4, 0x0 }, - { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x20 }, - { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 5, 0x0 }, - { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x18 }, + { 0xc7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x38 }, + { 0x65, BPF_REG_4, BPF_REG_0, (int16_t) 6, 0xffffffff }, + { 0xbf, BPF_REG_3, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xe }, + { 0x2d, BPF_REG_3, BPF_REG_2, (int16_t) 116, 0x0 }, + { 0xb7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xe }, + { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 13, 0x0 }, + { 0x55, BPF_REG_4, BPF_REG_0, (int16_t) 77, 0x14 }, + { 0xf, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x14 }, + { 0x2d, BPF_REG_3, BPF_REG_2, (int16_t) 109, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_1, (int16_t) 13, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x71, BPF_REG_2, BPF_REG_1, (int16_t) 12, 0x0 }, + { 0x4f, BPF_REG_3, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_2, BPF_REG_1, (int16_t) 15, 0x0 }, + { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 14, 0x0 }, + { 0x4f, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_5, BPF_REG_1, (int16_t) 9, 0x0 }, + { 0x67, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 8, 0x0 }, + { 0x4f, BPF_REG_5, BPF_REG_4, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 11, 0x0 }, + { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x71, BPF_REG_0, BPF_REG_1, (int16_t) 10, 0x0 }, + { 0x4f, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x10 }, + { 0x4f, BPF_REG_4, BPF_REG_5, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x10 }, + { 0x4f, BPF_REG_2, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_1, (int16_t) 17, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x71, BPF_REG_5, BPF_REG_1, (int16_t) 16, 0x0 }, + { 0x4f, BPF_REG_3, BPF_REG_5, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_5, BPF_REG_1, (int16_t) 19, 0x0 }, + { 0x67, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x71, BPF_REG_0, BPF_REG_1, (int16_t) 18, 0x0 }, + { 0x4f, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x4f, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x10 }, + { 0x4f, BPF_REG_5, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 1, 0x0 }, + { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x71, BPF_REG_3, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x4f, BPF_REG_4, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_1, (int16_t) 3, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x71, BPF_REG_0, BPF_REG_1, (int16_t) 2, 0x0 }, + { 0x4f, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x63, BPF_REG_10, BPF_REG_5, (int16_t) 65520, 0x0 }, + { 0x7b, BPF_REG_10, BPF_REG_2, (int16_t) 65512, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x10 }, { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 6, 0x0 }, - { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x10 }, - { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 7, 0x0 }, - { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x8 }, - { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_2, (int16_t) 8, 0x0 }, - { 0x4f, BPF_REG_3, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x7b, BPF_REG_10, BPF_REG_3, (int16_t) 65528, 0x0 }, + { 0x71, BPF_REG_2, BPF_REG_1, (int16_t) 5, 0x0 }, + { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 4, 0x0 }, + { 0x4f, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 6, 0x0 }, + { 0x71, BPF_REG_1, BPF_REG_1, (int16_t) 7, 0x0 }, + { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x4f, BPF_REG_1, BPF_REG_4, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x10 }, + { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x7b, BPF_REG_10, BPF_REG_1, (int16_t) 65504, 0x0 }, { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff8 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xffffffe0 }, + { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, { 0x18, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x0 }, { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 }, { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x52 }, - { 0xb7, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x67, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 41, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xfffffffe }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x1d, BPF_REG_0, BPF_REG_1, (int16_t) 2, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 36, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x63, BPF_REG_10, BPF_REG_7, (int16_t) 65532, 0x0 }, + { 0xbf, BPF_REG_2, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xfffffffc }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 28, 0x0 }, + { 0x61, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x61, BPF_REG_2, BPF_REG_6, (int16_t) 32, 0x0 }, + { 0x9f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x63, BPF_REG_10, BPF_REG_2, (int16_t) 65528, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff8 }, + { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, + { 0x18, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x52 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x67, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 13, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xfffffffe }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x1d, BPF_REG_0, BPF_REG_1, (int16_t) 1, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 65507, 0x0 }, + { 0xbf, BPF_REG_2, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xfffffffc }, + { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff8 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x2 }, + { 0xbf, BPF_REG_0, BPF_REG_7, (int16_t) 0, 0x0 }, { 0x95, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, }; @@ -86,3 +161,4 @@ ngx_bpf_program_t ngx_quic_reuseport_hel .license = "BSD", .type = BPF_PROG_TYPE_SK_REUSEPORT, }; + diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -36,6 +36,9 @@ typedef struct ngx_quic_keys_s ng #include #include #include +#if (NGX_QUIC_BPF) +#include +#endif /* RFC 9002, 6.2.2. Handshakes and New Paths: kInitialRtt */ @@ -45,6 +48,8 @@ typedef struct ngx_quic_keys_s ng #define NGX_QUIC_SEND_CTX_LAST (NGX_QUIC_ENCRYPTION_LAST - 1) +#define NGX_QUIC_MAX_SERVER_IDS 8 + /* 0-RTT and 1-RTT data exist in the same packet number space, * so we have 3 packet number spaces: * @@ -257,6 +262,7 @@ struct ngx_quic_connection_s { unsigned key_phase:1; unsigned validated:1; unsigned client_tp_done:1; + unsigned listen_bound:1; }; diff --git a/src/event/quic/ngx_event_quic_connid.c b/src/event/quic/ngx_event_quic_connid.c --- a/src/event/quic/ngx_event_quic_connid.c +++ b/src/event/quic/ngx_event_quic_connid.c @@ -9,12 +9,7 @@ #include #include -#define NGX_QUIC_MAX_SERVER_IDS 8 - -#if (NGX_QUIC_BPF) -static ngx_int_t ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id); -#endif static ngx_int_t ngx_quic_retire_client_id(ngx_connection_t *c, ngx_quic_client_id_t *cid); static ngx_quic_client_id_t *ngx_quic_alloc_client_id(ngx_connection_t *c, @@ -30,46 +25,10 @@ ngx_quic_create_server_id(ngx_connection return NGX_ERROR; } -#if (NGX_QUIC_BPF) - if (ngx_quic_bpf_attach_id(c, id) != NGX_OK) { - ngx_log_error(NGX_LOG_ERR, c->log, 0, - "quic bpf failed to generate socket key"); - /* ignore error, things still may work */ - } -#endif - return NGX_OK; } -#if (NGX_QUIC_BPF) - -static ngx_int_t -ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id) -{ - int fd; - uint64_t cookie; - socklen_t optlen; - - fd = c->listening->fd; - - optlen = sizeof(cookie); - - if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) { - ngx_log_error(NGX_LOG_ERR, c->log, ngx_socket_errno, - "quic getsockopt(SO_COOKIE) failed"); - - return NGX_ERROR; - } - - ngx_quic_dcid_encode_key(id, cookie); - - return NGX_OK; -} - -#endif - - ngx_int_t ngx_quic_handle_new_connection_id_frame(ngx_connection_t *c, ngx_quic_new_conn_id_frame_t *f) diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -84,6 +84,10 @@ ngx_quic_output(ngx_connection_t *c) ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; + if (c->fd == (ngx_socket_t) -1) { + return NGX_ERROR; + } + c->log->action = "sending frames"; qc = ngx_quic_get_connection(c); @@ -1031,7 +1035,6 @@ ngx_quic_send_retry(ngx_connection_t *c, pkt.odcid = inpkt->dcid; pkt.dcid = inpkt->scid; - /* TODO: generate routable dcid */ if (RAND_bytes(dcid, NGX_QUIC_SERVER_CID_LEN) != 1) { return NGX_ERROR; } diff --git a/src/event/quic/ngx_event_quic_socket.c b/src/event/quic/ngx_event_quic_socket.c --- a/src/event/quic/ngx_event_quic_socket.c +++ b/src/event/quic/ngx_event_quic_socket.c @@ -109,6 +109,13 @@ ngx_quic_open_sockets(ngx_connection_t * failed: ngx_rbtree_delete(&c->listening->rbtree, &qsock->udp.node); + +#if (NGX_QUIC_BPF) + if (ngx_quic_bpf_delete(c, qc, qsock) != NGX_OK) { + return NGX_ERROR; + } +#endif + c->udp = NULL; return NGX_ERROR; @@ -160,6 +167,13 @@ ngx_quic_close_socket(ngx_connection_t * ngx_queue_insert_head(&qc->free_sockets, &qsock->queue); ngx_rbtree_delete(&c->listening->rbtree, &qsock->udp.node); + +#if (NGX_QUIC_BPF) + if (ngx_quic_bpf_delete(c, qc, qsock) != NGX_OK) { + return; + } +#endif + qc->nsockets--; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -185,6 +199,12 @@ ngx_quic_listen(ngx_connection_t *c, ngx ngx_rbtree_insert(&c->listening->rbtree, &qsock->udp.node); +#if (NGX_QUIC_BPF) + if (ngx_quic_bpf_insert(c, qc, qsock) != NGX_OK) { + return NGX_ERROR; + } +#endif + ngx_queue_insert_tail(&qc->sockets, &qsock->queue); qc->nsockets++; diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c --- a/src/event/quic/ngx_event_quic_udp.c +++ b/src/event/quic/ngx_event_quic_udp.c @@ -160,7 +160,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) c->log->handler = NULL; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic recvmsg: fd:%d n:%z", c->fd, n); + "quic recvmsg: fd:%d n:%z", lc->fd, n); c->log->handler = handler; } @@ -193,12 +193,23 @@ ngx_quic_recvmsg(ngx_event_t *ev) ngx_accept_disabled = ngx_cycle->connection_n / 8 - ngx_cycle->free_connection_n; - c = ngx_get_connection(lc->fd, ev->log); - if (c == NULL) { + c = NULL; + +#if (NGX_QUIC_BPF) + if (ngx_quic_bpf_get_client_connection(lc, &c) != NGX_OK) { return; } +#endif - c->shared = 1; + if (c == NULL) { + c = ngx_get_connection(lc->fd, ev->log); + if (c == NULL) { + return; + } + + c->shared = 1; + } + c->type = SOCK_DGRAM; c->socklen = socklen; @@ -309,7 +320,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0, "*%uA quic recvmsg: %V fd:%d n:%z", - c->number, &addr, c->fd, n); + c->number, &addr, lc->fd, n); } } diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c --- a/src/os/unix/ngx_process_cycle.c +++ b/src/os/unix/ngx_process_cycle.c @@ -955,7 +955,8 @@ ngx_worker_process_exit(ngx_cycle_t *cyc && c[i].read && !c[i].read->accept && !c[i].read->channel - && !c[i].read->resolver) + && !c[i].read->resolver + && !c[i].read->quic) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "*%uA open socket #%d left in connection %ui", From arut at nginx.com Tue Jan 17 11:24:02 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Tue, 17 Jan 2023 11:24:02 +0000 Subject: [PATCH 4 of 4] QUIC: client sockets In-Reply-To: References: Message-ID: <7873810feab00761677f.1673954642@ip-10-1-18-114.eu-central-1.compute.internal> # HG changeset patch # User Roman Arutyunyan # Date 1673868053 0 # Mon Jan 16 11:20:53 2023 +0000 # Branch quic # Node ID 7873810feab00761677f02224a2f6c1b0f130203 # Parent 52681b76d4db0e9e814f1e69b0d6b35f5ac630a2 QUIC: client sockets. In client sockets mode, a new socket is created for each client. This socket is bound to server address and connected to client address. This allows for seamless configuration reload and binary upgrade. This mode is enabled by default and can be disabled by "--without-quic_client_sockets" configure option. With this approach, it is possible that some new connection packets will arrive not to the listen socket, but to a client socket due to a race between bind() and connect(). Such packets initiate new connections in the workers they end up in. diff --git a/auto/modules b/auto/modules --- a/auto/modules +++ b/auto/modules @@ -1378,6 +1378,10 @@ if [ $USE_OPENSSL_QUIC = YES ]; then have=NGX_QUIC_BPF . auto/have fi + + if [ $QUIC_CLIENT_SOCKETS = YES ]; then + have=NGX_QUIC_CLIENT_SOCKETS . auto/have + fi fi diff --git a/auto/options b/auto/options --- a/auto/options +++ b/auto/options @@ -45,6 +45,7 @@ USE_THREADS=NO NGX_FILE_AIO=NO +QUIC_CLIENT_SOCKETS=YES QUIC_BPF=NO HTTP=YES @@ -216,6 +217,7 @@ do --with-file-aio) NGX_FILE_AIO=YES ;; + --without-quic_client_sockets) QUIC_CLIENT_SOCKETS=NONE ;; --without-quic_bpf_module) QUIC_BPF=NONE ;; --with-ipv6) @@ -452,6 +454,7 @@ cat << END --with-file-aio enable file AIO support + --without-quic_client_sockets disable QUIC client sockets --without-quic_bpf_module disable ngx_quic_bpf_module --with-http_ssl_module enable ngx_http_ssl_module diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -513,6 +513,33 @@ ngx_open_listening_sockets(ngx_cycle_t * #if (NGX_HAVE_REUSEPORT) +#if (NGX_QUIC_CLIENT_SOCKETS && defined SO_REUSEPORT_LB) + + if (ls[i].quic) { + int reuseport; + + reuseport = 1; + + if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, + (const void *) &reuseport, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, + "setsockopt(SO_REUSEPORT) %V failed", + &ls[i].addr_text); + + if (ngx_close_socket(s) == -1) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, + ngx_close_socket_n " %V failed", + &ls[i].addr_text); + } + + return NGX_ERROR; + } + } + +#endif + if (ls[i].reuseport && !ngx_test_config) { int reuseport; diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -10,11 +10,15 @@ #include +#define NGX_QUIC_LINGERING_TIMEOUT 50 + + static ngx_quic_connection_t *ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, ngx_quic_header_t *pkt); static ngx_int_t ngx_quic_handle_stateless_reset(ngx_connection_t *c, ngx_quic_header_t *pkt); static void ngx_quic_input_handler(ngx_event_t *rev); +static void ngx_quic_lingering_handler(ngx_event_t *rev); static void ngx_quic_close_handler(ngx_event_t *ev); static ngx_int_t ngx_quic_handle_packet(ngx_connection_t *c, @@ -208,6 +212,13 @@ ngx_quic_run(ngx_connection_t *c, ngx_qu ngx_quic_close_connection(c, rc); return; } + + if (!c->shared) { + if (ngx_add_event(c->read, NGX_READ_EVENT, 0) == NGX_ERROR) { + ngx_quic_close_connection(c, NGX_ERROR); + return; + } + } } @@ -432,6 +443,10 @@ ngx_quic_input_handler(ngx_event_t *rev) return; } + + if (!c->shared) { + ngx_quic_recvmsg(rev); + } } @@ -574,6 +589,15 @@ quic_done: c->destroyed = 1; + if (c->read->ready) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic lingering"); + + c->read->handler = ngx_quic_lingering_handler; + ngx_add_timer(c->read, NGX_QUIC_LINGERING_TIMEOUT); + + return; + } + pool = c->pool; ngx_close_connection(c); @@ -582,6 +606,31 @@ quic_done: } +static void +ngx_quic_lingering_handler(ngx_event_t *rev) +{ + ngx_pool_t *pool; + ngx_connection_t *c; + + c = rev->data; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic lingering handler"); + + if (rev->ready && !rev->timedout) { + ngx_quic_recvmsg(rev); + } + + if (!rev->ready || rev->timedout) { + pool = c->pool; + + ngx_close_connection(c); + ngx_destroy_pool(pool); + + return; + } +} + + void ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err, const char *reason) diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -437,8 +437,10 @@ ngx_quic_send_segments(ngx_connection_t msg.msg_iov = &iov; msg.msg_iovlen = 1; - msg.msg_name = sockaddr; - msg.msg_namelen = socklen; + if (c->shared) { + msg.msg_name = sockaddr; + msg.msg_namelen = socklen; + } msg.msg_control = msg_control; msg.msg_controllen = sizeof(msg_control); @@ -455,7 +457,7 @@ ngx_quic_send_segments(ngx_connection_t *valp = segment; #if (NGX_HAVE_ADDRINFO_CMSG) - if (c->listening && c->listening->wildcard && c->local_sockaddr) { + if (c->shared && c->listening->wildcard) { cmsg = CMSG_NXTHDR(&msg, cmsg); clen += ngx_set_srcaddr_cmsg(cmsg, c->local_sockaddr); } @@ -747,11 +749,13 @@ ngx_quic_send(ngx_connection_t *c, u_cha msg.msg_iov = &iov; msg.msg_iovlen = 1; - msg.msg_name = sockaddr; - msg.msg_namelen = socklen; + if (c->shared) { + msg.msg_name = sockaddr; + msg.msg_namelen = socklen; + } #if (NGX_HAVE_ADDRINFO_CMSG) - if (c->listening && c->listening->wildcard && c->local_sockaddr) { + if (c->shared && c->listening->wildcard) { msg.msg_control = msg_control; msg.msg_controllen = sizeof(msg_control); diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c --- a/src/event/quic/ngx_event_quic_udp.c +++ b/src/event/quic/ngx_event_quic_udp.c @@ -11,6 +11,11 @@ #include +#if (NGX_QUIC_CLIENT_SOCKETS) +static ngx_int_t ngx_quic_create_client_socket_connection(ngx_connection_t *lc, + ngx_connection_t **pc, struct sockaddr *sockaddr, socklen_t socklen, + struct sockaddr *local_sockaddr, socklen_t local_socklen); +#endif static void ngx_quic_close_accepted_connection(ngx_connection_t *c); static ngx_connection_t *ngx_quic_lookup_connection(ngx_listening_t *ls, ngx_str_t *key, struct sockaddr *local_sockaddr, socklen_t local_socklen); @@ -48,7 +53,6 @@ ngx_quic_recvmsg(ngx_event_t *ev) lc = ev->data; ls = lc->listening; - ev->ready = 0; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, "quic recvmsg on %V, ready: %d", @@ -65,8 +69,17 @@ ngx_quic_recvmsg(ngx_event_t *ev) msg.msg_iov = iov; msg.msg_iovlen = 1; + if (lc->local_sockaddr) { + local_sockaddr = lc->local_sockaddr; + local_socklen = lc->local_socklen; + + } else { + local_sockaddr = ls->sockaddr; + local_socklen = ls->socklen; + } + #if (NGX_HAVE_ADDRINFO_CMSG) - if (ls->wildcard) { + if (ngx_inet_wildcard(local_sockaddr)) { msg.msg_control = &msg_control; msg.msg_controllen = sizeof(msg_control); @@ -82,6 +95,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) if (err == NGX_EAGAIN) { ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err, "quic recvmsg() not ready"); + ev->ready = 0; return; } @@ -121,12 +135,9 @@ ngx_quic_recvmsg(ngx_event_t *ev) #endif - local_sockaddr = ls->sockaddr; - local_socklen = ls->socklen; - #if (NGX_HAVE_ADDRINFO_CMSG) - if (ls->wildcard) { + if (ngx_inet_wildcard(local_sockaddr)) { struct cmsghdr *cmsg; ngx_memcpy(&lsa, local_sockaddr, local_socklen); @@ -201,6 +212,17 @@ ngx_quic_recvmsg(ngx_event_t *ev) } #endif +#if (NGX_QUIC_CLIENT_SOCKETS) + if (c == NULL) { + if (ngx_quic_create_client_socket_connection(lc, &c, + sockaddr, socklen, local_sockaddr, local_socklen) + != NGX_OK) + { + return; + } + } +#endif + if (c == NULL) { c = ngx_get_connection(lc->fd, ev->log); if (c == NULL) { @@ -243,17 +265,13 @@ ngx_quic_recvmsg(ngx_event_t *ev) c->pool->log = log; c->listening = ls; - if (local_sockaddr == &lsa.sockaddr) { - local_sockaddr = ngx_palloc(c->pool, local_socklen); - if (local_sockaddr == NULL) { - ngx_quic_close_accepted_connection(c); - return; - } - - ngx_memcpy(local_sockaddr, &lsa, local_socklen); + c->local_sockaddr = ngx_palloc(c->pool, local_socklen); + if (c->local_sockaddr == NULL) { + ngx_quic_close_accepted_connection(c); + return; } - c->local_sockaddr = local_sockaddr; + ngx_memcpy(c->local_sockaddr, local_sockaddr, local_socklen); c->local_socklen = local_socklen; c->buffer = ngx_create_temp_buf(c->pool, n); @@ -267,7 +285,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) rev = c->read; wev = c->write; - rev->active = 1; + rev->ready = (c->shared ? 0 : 1); wev->ready = 1; rev->log = log; @@ -341,6 +359,94 @@ ngx_quic_recvmsg(ngx_event_t *ev) } +#if (NGX_QUIC_CLIENT_SOCKETS) + +static ngx_int_t +ngx_quic_create_client_socket_connection(ngx_connection_t *lc, + ngx_connection_t **pc, struct sockaddr *sockaddr, socklen_t socklen, + struct sockaddr *local_sockaddr, socklen_t local_socklen) +{ + int value; + ngx_socket_t s; + + s = ngx_socket(sockaddr->sa_family, SOCK_DGRAM, 0); + if (s == (ngx_socket_t) -1) { + ngx_log_error(NGX_LOG_ERR, lc->log, ngx_socket_errno, + ngx_socket_n " failed"); + return NGX_ERROR; + } + + if (ngx_nonblocking(s) == -1) { + ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno, + ngx_nonblocking_n " client socket failed"); + goto failed; + } + + value = 1; + +#if (NGX_HAVE_REUSEPORT && !NGX_LINUX) + + if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno, + "setsockopt(SO_REUSEPORT) client socket failed"); + goto failed; + } + +#else + + if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno, + "setsockopt(SO_REUSEADDR) client socket failed"); + goto failed; + } + +#endif + + if (bind(s, local_sockaddr, local_socklen) == -1) { + ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno, + "bind() for client socket failed"); + goto failed; + } + + if (connect(s, sockaddr, socklen) == -1) { + ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno, + "connect() to client failed"); + goto failed; + } + + *pc = ngx_get_connection(s, lc->log); + if (*pc == NULL) { + goto failed; + } + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, lc->log, 0, + "quic client socket connection fd:%d r:%d", + s, lc->listening->connection != lc); + + ngx_accept_disabled = ngx_cycle->connection_n / 8 + - ngx_cycle->free_connection_n; + + return NGX_OK; + +failed: + + if (ngx_close_socket(s) == -1) { + ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno, + ngx_close_socket_n " client failed"); + } + + return NGX_ERROR; +} + +#endif + + static void ngx_quic_close_accepted_connection(ngx_connection_t *c) { From pluknet at nginx.com Tue Jan 17 12:01:39 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Tue, 17 Jan 2023 16:01:39 +0400 Subject: [PATCH] QUIC: defer setting the active flag for client stream events In-Reply-To: <4EB02BE1-CF83-4ED5-9282-FA5D26651ADE@nginx.com> References: <4EB02BE1-CF83-4ED5-9282-FA5D26651ADE@nginx.com> Message-ID: > On 17 Jan 2023, at 14:44, Roman Arutyunyan wrote: > > > >> On 16 Jan 2023, at 17:27, Sergey Kandaurov wrote: >> >> # HG changeset patch >> # User Sergey Kandaurov >> # Date 1673875616 -14400 >> # Mon Jan 16 17:26:56 2023 +0400 >> # Branch quic >> # Node ID f7c7cabe232898db5b16a142163de71964cebcfd >> # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 >> QUIC: defer setting the active flag for client stream events. >> >> Specifically, now it is kept unset until streams are initialized. >> Notably, this unbreaks OCSP with client certificates after 35e27117b593. >> Previously, the read event could be posted prematurely in ngx_quic_set_event() >> e.g., as part of handling a STREAM frame. >> >> diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c >> --- a/src/event/quic/ngx_event_quic_streams.c >> +++ b/src/event/quic/ngx_event_quic_streams.c >> @@ -106,6 +106,13 @@ ngx_quic_open_stream(ngx_connection_t *c >> return NULL; >> } >> >> + nqs->connection->write->active = 1; >> + nqs->connection->write->ready = 1; >> + >> + if (!bidi) { > > if (bidi) ? Sure, tnx. > >> + nqs->connection->read->active = 1; >> + } >> + >> return nqs->connection; > > This could've been simplified by adding a variable for nqs->connection, YMMV. Agree, and it looks similar to other places. While here, renamed nqs to just a qs, the only one after 6434160b4b78. # HG changeset patch # User Sergey Kandaurov # Date 1673956193 -14400 # Tue Jan 17 15:49:53 2023 +0400 # Branch quic # Node ID fa3018f0102247215953126515974c76d2b02e53 # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 QUIC: defer setting the active flag for client stream events. Specifically, now it is kept unset until streams are initialized. Notably, this unbreaks OCSP with client certificates after 35e27117b593. Previously, the read event could be posted prematurely via ngx_quic_set_event() e.g., as part of handling a STREAM frame. diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c --- a/src/event/quic/ngx_event_quic_streams.c +++ b/src/event/quic/ngx_event_quic_streams.c @@ -46,8 +46,8 @@ ngx_connection_t * ngx_quic_open_stream(ngx_connection_t *c, ngx_uint_t bidi) { uint64_t id; - ngx_connection_t *pc; - ngx_quic_stream_t *nqs; + ngx_connection_t *pc, *sc; + ngx_quic_stream_t *qs; ngx_quic_connection_t *qc; pc = c->quic ? c->quic->parent : c; @@ -101,12 +101,21 @@ ngx_quic_open_stream(ngx_connection_t *c qc->streams.server_streams_uni++; } - nqs = ngx_quic_create_stream(pc, id); - if (nqs == NULL) { + qs = ngx_quic_create_stream(pc, id); + if (qs == NULL) { return NULL; } - return nqs->connection; + sc = qs->connection; + + sc->write->active = 1; + sc->write->ready = 1; + + if (bidi) { + sc->read->active = 1; + } + + return sc; } @@ -534,6 +543,13 @@ ngx_quic_init_stream_handler(ngx_event_t ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic init stream"); + if ((qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0) { + c->write->active = 1; + c->write->ready = 1; + } + + c->read->active = 1; + ngx_queue_remove(&qs->queue); c->listening->handler(c); @@ -704,19 +720,6 @@ ngx_quic_create_stream(ngx_connection_t log->connection = sc->number; - if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0 - || (id & NGX_QUIC_STREAM_SERVER_INITIATED)) - { - sc->write->active = 1; - sc->write->ready = 1; - } - - if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0 - || (id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0) - { - sc->read->active = 1; - } - if (id & NGX_QUIC_STREAM_UNIDIRECTIONAL) { if (id & NGX_QUIC_STREAM_SERVER_INITIATED) { qs->send_max_data = qc->ctp.initial_max_stream_data_uni; -- Sergey Kandaurov From arut at nginx.com Tue Jan 17 12:18:41 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Tue, 17 Jan 2023 16:18:41 +0400 Subject: [PATCH] QUIC: defer setting the active flag for client stream events In-Reply-To: References: <4EB02BE1-CF83-4ED5-9282-FA5D26651ADE@nginx.com> Message-ID: <20230117121841.zftixrbaurwzblzg@N00W24XTQX> On Tue, Jan 17, 2023 at 04:01:39PM +0400, Sergey Kandaurov wrote: > > > On 17 Jan 2023, at 14:44, Roman Arutyunyan wrote: > > > > > > > >> On 16 Jan 2023, at 17:27, Sergey Kandaurov wrote: > >> > >> # HG changeset patch > >> # User Sergey Kandaurov > >> # Date 1673875616 -14400 > >> # Mon Jan 16 17:26:56 2023 +0400 > >> # Branch quic > >> # Node ID f7c7cabe232898db5b16a142163de71964cebcfd > >> # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 > >> QUIC: defer setting the active flag for client stream events. > >> > >> Specifically, now it is kept unset until streams are initialized. > >> Notably, this unbreaks OCSP with client certificates after 35e27117b593. > >> Previously, the read event could be posted prematurely in ngx_quic_set_event() > >> e.g., as part of handling a STREAM frame. > >> > >> diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c > >> --- a/src/event/quic/ngx_event_quic_streams.c > >> +++ b/src/event/quic/ngx_event_quic_streams.c > >> @@ -106,6 +106,13 @@ ngx_quic_open_stream(ngx_connection_t *c > >> return NULL; > >> } > >> > >> + nqs->connection->write->active = 1; > >> + nqs->connection->write->ready = 1; > >> + > >> + if (!bidi) { > > > > if (bidi) ? > > Sure, tnx. > > > > >> + nqs->connection->read->active = 1; > >> + } > >> + > >> return nqs->connection; > > > > This could've been simplified by adding a variable for nqs->connection, YMMV. > > Agree, and it looks similar to other places. > > While here, renamed nqs to just a qs, the only one after 6434160b4b78. > > # HG changeset patch > # User Sergey Kandaurov > # Date 1673956193 -14400 > # Tue Jan 17 15:49:53 2023 +0400 > # Branch quic > # Node ID fa3018f0102247215953126515974c76d2b02e53 > # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 > QUIC: defer setting the active flag for client stream events. > > Specifically, now it is kept unset until streams are initialized. > Notably, this unbreaks OCSP with client certificates after 35e27117b593. > Previously, the read event could be posted prematurely via ngx_quic_set_event() > e.g., as part of handling a STREAM frame. > > diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c > --- a/src/event/quic/ngx_event_quic_streams.c > +++ b/src/event/quic/ngx_event_quic_streams.c > @@ -46,8 +46,8 @@ ngx_connection_t * > ngx_quic_open_stream(ngx_connection_t *c, ngx_uint_t bidi) > { > uint64_t id; > - ngx_connection_t *pc; > - ngx_quic_stream_t *nqs; > + ngx_connection_t *pc, *sc; > + ngx_quic_stream_t *qs; > ngx_quic_connection_t *qc; > > pc = c->quic ? c->quic->parent : c; > @@ -101,12 +101,21 @@ ngx_quic_open_stream(ngx_connection_t *c > qc->streams.server_streams_uni++; > } > > - nqs = ngx_quic_create_stream(pc, id); > - if (nqs == NULL) { > + qs = ngx_quic_create_stream(pc, id); > + if (qs == NULL) { > return NULL; > } > > - return nqs->connection; > + sc = qs->connection; > + > + sc->write->active = 1; > + sc->write->ready = 1; > + > + if (bidi) { > + sc->read->active = 1; > + } > + > + return sc; > } > > > @@ -534,6 +543,13 @@ ngx_quic_init_stream_handler(ngx_event_t > > ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic init stream"); > > + if ((qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0) { > + c->write->active = 1; > + c->write->ready = 1; > + } > + > + c->read->active = 1; > + > ngx_queue_remove(&qs->queue); > > c->listening->handler(c); > @@ -704,19 +720,6 @@ ngx_quic_create_stream(ngx_connection_t > > log->connection = sc->number; > > - if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0 > - || (id & NGX_QUIC_STREAM_SERVER_INITIATED)) > - { > - sc->write->active = 1; > - sc->write->ready = 1; > - } > - > - if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0 > - || (id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0) > - { > - sc->read->active = 1; > - } > - > if (id & NGX_QUIC_STREAM_UNIDIRECTIONAL) { > if (id & NGX_QUIC_STREAM_SERVER_INITIATED) { > qs->send_max_data = qc->ctp.initial_max_stream_data_uni; > > > -- > Sergey Kandaurov > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel Looks fine -- Roman Arutyunyan From shankerwangmiao at gmail.com Wed Jan 18 15:28:52 2023 From: shankerwangmiao at gmail.com (Miao Wang) Date: Wed, 18 Jan 2023 23:28:52 +0800 Subject: [PATCH] HTTP: trigger lingering close when keepalive connection will be closed Message-ID: <699749D2-D96F-47D8-85DA-74E7F97682FF@gmail.com> # HG changeset patch # User Miao Wang # Date 1674055068 -28800 # Wed Jan 18 23:17:48 2023 +0800 # Node ID 73aa64bd29f3dec9e43e97560d6b5a07cdf40063 # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 HTTP: trigger lingering close when keepalive connection will be closed When finalizing a request, if the request is not keepalive but its connection has served more than one request, then the connection has been a keepalive connection previously and this connection will be closed after this response. In this condition, it is likely that there are pipelined requests following this request, which we should ignore. As a result, lingering close is necessary in this case. Without this patch, nginx (with its default configuration) will send out TCP RST when there are more pipelined requests. The symptom is obvious when nginx is serving a debian repository and apt is downloading massive of packages. See [1]. It becomes more obvious when `keepalive_requests` is lower or nginx is under a relative higher load, and it disappears when specifying `lingering_close always`. [1]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 diff -r 07b0bee87f32 -r 73aa64bd29f3 src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c Wed Dec 21 14:53:27 2022 +0300 +++ b/src/http/ngx_http_request.c Wed Jan 18 23:17:48 2023 +0800 @@ -2749,6 +2749,10 @@ return; } + if (!r->keepalive && r->connection->requests > 1) { + r->lingering_close = 1; + } + if (clcf->lingering_close == NGX_HTTP_LINGERING_ALWAYS || (clcf->lingering_close == NGX_HTTP_LINGERING_ON && (r->lingering_close From mdounin at mdounin.ru Wed Jan 18 19:50:23 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Wed, 18 Jan 2023 22:50:23 +0300 Subject: [PATCH] Fixed handling of very long locations (ticket #2435) Message-ID: <4af6cc78dc72fbb15326.1674071423@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1674071317 -10800 # Wed Jan 18 22:48:37 2023 +0300 # Node ID 4af6cc78dc72fbb15326e6ffbbd30935f0cb794b # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 Fixed handling of very long locations (ticket #2435). Previously, location prefix length in ngx_http_location_tree_node_t was stored as "u_char", and therefore location prefixes longer than 255 bytes were handled incorrectly. Fix is to use "u_short" instead. With "u_short", prefixes up to 65535 bytes can be safely used, and this isn't reachable due to NGX_CONF_BUFFER, which is 4096 bytes. diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c --- a/src/http/ngx_http.c +++ b/src/http/ngx_http.c @@ -1130,7 +1130,7 @@ ngx_http_create_locations_tree(ngx_conf_ node->auto_redirect = (u_char) ((lq->exact && lq->exact->auto_redirect) || (lq->inclusive && lq->inclusive->auto_redirect)); - node->len = (u_char) len; + node->len = (u_short) len; ngx_memcpy(node->name, &lq->name->data[prefix], len); ngx_queue_split(locations, q, &tail); diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -463,8 +463,8 @@ struct ngx_http_location_tree_node_s { ngx_http_core_loc_conf_t *exact; ngx_http_core_loc_conf_t *inclusive; + u_short len; u_char auto_redirect; - u_char len; u_char name[1]; }; From xeioex at nginx.com Thu Jan 19 04:06:56 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 19 Jan 2023 04:06:56 +0000 Subject: [njs] Added njs_vm_add_module(). Message-ID: details: https://hg.nginx.org/njs/rev/6bcfa2934487 branches: changeset: 2025:6bcfa2934487 user: Dmitry Volyntsev date: Fri Jan 06 20:39:01 2023 -0800 description: Added njs_vm_add_module(). The new API does not require for njs_mod_t structure to be disclosed. diffstat: external/njs_crypto_module.c | 6 ++---- external/njs_fs_module.c | 5 +---- external/njs_query_string_module.c | 6 ++---- src/njs.h | 2 ++ src/njs_buffer.c | 6 ++---- src/njs_module.c | 17 +++++++++-------- src/njs_module.h | 2 +- src/njs_vm.c | 9 ++++++++- 8 files changed, 27 insertions(+), 26 deletions(-) diffs (162 lines): diff -r a98b63e87688 -r 6bcfa2934487 external/njs_crypto_module.c --- a/external/njs_crypto_module.c Fri Jan 06 16:50:46 2023 -0800 +++ b/external/njs_crypto_module.c Fri Jan 06 20:39:01 2023 -0800 @@ -646,13 +646,11 @@ njs_crypto_init(njs_vm_t *vm) return NJS_ERROR; } - module = njs_module_add(vm, &njs_str_value("crypto")); + module = njs_vm_add_module(vm, &njs_str_value("crypto"), + njs_value_arg(&value)); if (njs_slow_path(module == NULL)) { return NJS_ERROR; } - njs_value_assign(&module->value, &value); - module->function.native = 1; - return NJS_OK; } diff -r a98b63e87688 -r 6bcfa2934487 external/njs_fs_module.c --- a/external/njs_fs_module.c Fri Jan 06 16:50:46 2023 -0800 +++ b/external/njs_fs_module.c Fri Jan 06 20:39:01 2023 -0800 @@ -3762,13 +3762,10 @@ njs_fs_init(njs_vm_t *vm) return NJS_ERROR; } - module = njs_module_add(vm, &njs_str_value("fs")); + module = njs_vm_add_module(vm, &njs_str_value("fs"), njs_value_arg(&value)); if (njs_slow_path(module == NULL)) { return NJS_ERROR; } - njs_value_assign(&module->value, &value); - module->function.native = 1; - return NJS_OK; } diff -r a98b63e87688 -r 6bcfa2934487 external/njs_query_string_module.c --- a/external/njs_query_string_module.c Fri Jan 06 16:50:46 2023 -0800 +++ b/external/njs_query_string_module.c Fri Jan 06 20:39:01 2023 -0800 @@ -998,13 +998,11 @@ njs_query_string_init(njs_vm_t *vm) return NJS_ERROR; } - module = njs_module_add(vm, &njs_str_value("querystring")); + module = njs_vm_add_module(vm, &njs_str_value("querystring"), + njs_value_arg(&value)); if (njs_slow_path(module == NULL)) { return NJS_ERROR; } - njs_value_assign(&module->value, &value); - module->function.native = 1; - return NJS_OK; } diff -r a98b63e87688 -r 6bcfa2934487 src/njs.h --- a/src/njs.h Fri Jan 06 16:50:46 2023 -0800 +++ b/src/njs.h Fri Jan 06 20:39:01 2023 -0800 @@ -310,6 +310,8 @@ NJS_EXPORT njs_vm_t *njs_vm_create(njs_v NJS_EXPORT void njs_vm_destroy(njs_vm_t *vm); NJS_EXPORT njs_int_t njs_vm_compile(njs_vm_t *vm, u_char **start, u_char *end); +NJS_EXPORT njs_mod_t *njs_vm_add_module(njs_vm_t *vm, njs_str_t *name, + njs_value_t *value); NJS_EXPORT njs_mod_t *njs_vm_compile_module(njs_vm_t *vm, njs_str_t *name, u_char **start, u_char *end); NJS_EXPORT njs_vm_t *njs_vm_clone(njs_vm_t *vm, njs_external_ptr_t external); diff -r a98b63e87688 -r 6bcfa2934487 src/njs_buffer.c --- a/src/njs_buffer.c Fri Jan 06 16:50:46 2023 -0800 +++ b/src/njs_buffer.c Fri Jan 06 20:39:01 2023 -0800 @@ -2602,13 +2602,11 @@ njs_buffer_init(njs_vm_t *vm) return NJS_ERROR; } - module = njs_module_add(vm, &njs_str_value("buffer")); + module = njs_vm_add_module(vm, &njs_str_value("buffer"), + njs_value_arg(&value)); if (njs_slow_path(module == NULL)) { return NJS_ERROR; } - njs_value_assign(&module->value, &value); - module->function.native = 1; - return NJS_OK; } diff -r a98b63e87688 -r 6bcfa2934487 src/njs_module.c --- a/src/njs_module.c Fri Jan 06 16:50:46 2023 -0800 +++ b/src/njs_module.c Fri Jan 06 20:39:01 2023 -0800 @@ -283,7 +283,7 @@ njs_module_find(njs_vm_t *vm, njs_str_t njs_mod_t * -njs_module_add(njs_vm_t *vm, njs_str_t *name) +njs_module_add(njs_vm_t *vm, njs_str_t *name, njs_value_t *value) { njs_int_t ret; njs_mod_t *module; @@ -309,16 +309,17 @@ njs_module_add(njs_vm_t *vm, njs_str_t * lhq.proto = &njs_modules_hash_proto; ret = njs_lvlhsh_insert(&vm->shared->modules_hash, &lhq); - if (njs_fast_path(ret == NJS_OK)) { - return module; + if (njs_slow_path(ret != NJS_OK)) { + njs_internal_error(vm, "lvlhsh insert failed"); + return NULL; } - njs_mp_free(vm->mem_pool, module->name.start); - njs_mp_free(vm->mem_pool, module); + if (value != NULL) { + njs_value_assign(&module->value, value); + module->function.native = 1; + } - njs_internal_error(vm, "lvlhsh insert failed"); - - return NULL; + return module; } diff -r a98b63e87688 -r 6bcfa2934487 src/njs_module.h --- a/src/njs_module.h Fri Jan 06 16:50:46 2023 -0800 +++ b/src/njs_module.h Fri Jan 06 20:39:01 2023 -0800 @@ -16,7 +16,7 @@ struct njs_mod_s { }; -njs_mod_t *njs_module_add(njs_vm_t *vm, njs_str_t *name); +njs_mod_t *njs_module_add(njs_vm_t *vm, njs_str_t *name, njs_value_t *value); njs_mod_t *njs_module_find(njs_vm_t *vm, njs_str_t *name, njs_bool_t shared); njs_mod_t *njs_parser_module(njs_parser_t *parser, njs_str_t *name); diff -r a98b63e87688 -r 6bcfa2934487 src/njs_vm.c --- a/src/njs_vm.c Fri Jan 06 16:50:46 2023 -0800 +++ b/src/njs_vm.c Fri Jan 06 20:39:01 2023 -0800 @@ -239,6 +239,13 @@ njs_vm_compile(njs_vm_t *vm, u_char **st njs_mod_t * +njs_vm_add_module(njs_vm_t *vm, njs_str_t *name, njs_value_t *value) +{ + return njs_module_add(vm, name, value); +} + + +njs_mod_t * njs_vm_compile_module(njs_vm_t *vm, njs_str_t *name, u_char **start, u_char *end) { @@ -256,7 +263,7 @@ njs_vm_compile_module(njs_vm_t *vm, njs_ return module; } - module = njs_module_add(vm, name); + module = njs_module_add(vm, name, NULL); if (njs_slow_path(module == NULL)) { return NULL; } From xeioex at nginx.com Thu Jan 19 04:06:57 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 19 Jan 2023 04:06:57 +0000 Subject: [njs] Shell: increased max output size for console.log() to 32768. Message-ID: details: https://hg.nginx.org/njs/rev/8b9414a6e557 branches: changeset: 2026:8b9414a6e557 user: Dmitry Volyntsev date: Wed Jan 18 18:33:23 2023 -0800 description: Shell: increased max output size for console.log() to 32768. diffstat: src/njs_vm.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 6bcfa2934487 -r 8b9414a6e557 src/njs_vm.c --- a/src/njs_vm.c Fri Jan 06 20:39:01 2023 -0800 +++ b/src/njs_vm.c Wed Jan 18 18:33:23 2023 -0800 @@ -906,7 +906,7 @@ njs_vm_logger(njs_vm_t *vm, njs_log_leve u_char *p; va_list args; njs_logger_t logger; - u_char buf[NJS_MAX_ERROR_STR]; + u_char buf[32768]; if (vm->options.ops == NULL) { return; From arut at nginx.com Thu Jan 19 14:01:13 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Thu, 19 Jan 2023 18:01:13 +0400 Subject: [PATCH 1 of 6] QUIC: ignore server address while looking up a connection In-Reply-To: References: <1038d7300c29eea02b47.1670578727@ip-10-1-18-114.eu-central-1.compute.internal> Message-ID: <20230119140113.3i7kpmjnuzfppjjh@N00W24XTQX> Hi, On Thu, Dec 29, 2022 at 05:13:49PM +0400, Sergey Kandaurov wrote: [..] > # HG changeset patch > # User Yu Zhu > # Date 1672317960 -14400 > # Thu Dec 29 16:46:00 2022 +0400 > # Branch quic > # Node ID 46e738a5c0ab9d98577b21e72447cc9a6e3e9784 > # Parent 91ad1abfb2850f952bccb607e4c5843854576a09 > QUIC: moved rtt and congestion control to ngx_quic_path_t. > > As per RFC 9002, section 6. Loss Detection: > > Loss detection is separate per packet number space, unlike RTT measurement > and congestion control, because RTT and congestion control are properties > of the path, whereas loss detection also relies upon key availability. RFC 9000 (Section 9.4.) is less certain about the congestion control part: While multiple paths might be used during connection migration, a single congestion control context and a single loss recovery context (as described in [QUIC-RECOVERY]) could be adequate. For instance, an endpoint might delay switching to a new congestion control context until it is confirmed that an old path is no longer needed (such as the case described in Section 9.3.3). Loss recovery context is path-independent before and after the change, so maybe congestion control should remain where it is now. > No functional changes. I'm not sure about this. > diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c > --- a/src/event/quic/ngx_event_quic.c > +++ b/src/event/quic/ngx_event_quic.c > @@ -263,15 +263,6 @@ ngx_quic_new_connection(ngx_connection_t > > ngx_queue_init(&qc->free_frames); > > - qc->avg_rtt = NGX_QUIC_INITIAL_RTT; > - qc->rttvar = NGX_QUIC_INITIAL_RTT / 2; > - qc->min_rtt = NGX_TIMER_INFINITE; > - qc->first_rtt = NGX_TIMER_INFINITE; > - > - /* > - * qc->latest_rtt = 0 > - */ > - > qc->pto.log = c->log; > qc->pto.data = c; > qc->pto.handler = ngx_quic_pto_handler; > @@ -311,12 +302,6 @@ ngx_quic_new_connection(ngx_connection_t > qc->streams.client_max_streams_uni = qc->tp.initial_max_streams_uni; > qc->streams.client_max_streams_bidi = qc->tp.initial_max_streams_bidi; > > - qc->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size, > - ngx_max(2 * qc->tp.max_udp_payload_size, > - 14720)); > - qc->congestion.ssthresh = (size_t) -1; > - qc->congestion.recovery_start = ngx_current_msec; > - > if (pkt->validated && pkt->retried) { > qc->tp.retry_scid.len = pkt->dcid.len; > qc->tp.retry_scid.data = ngx_pstrdup(c->pool, &pkt->dcid); > diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c > --- a/src/event/quic/ngx_event_quic_ack.c > +++ b/src/event/quic/ngx_event_quic_ack.c > @@ -29,7 +29,7 @@ typedef struct { > } ngx_quic_ack_stat_t; > > > -static ngx_inline ngx_msec_t ngx_quic_lost_threshold(ngx_quic_connection_t *qc); > +static ngx_inline ngx_msec_t ngx_quic_lost_threshold(ngx_quic_path_t *path); > static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack, > enum ssl_encryption_level_t level, ngx_msec_t send_time); > static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c, > @@ -48,11 +48,11 @@ static void ngx_quic_lost_handler(ngx_ev > > /* RFC 9002, 6.1.2. Time Threshold: kTimeThreshold, kGranularity */ > static ngx_inline ngx_msec_t > -ngx_quic_lost_threshold(ngx_quic_connection_t *qc) > +ngx_quic_lost_threshold(ngx_quic_path_t *path) > { > ngx_msec_t thr; > > - thr = ngx_max(qc->latest_rtt, qc->avg_rtt); > + thr = ngx_max(path->latest_rtt, path->avg_rtt); > thr += thr >> 3; > > return ngx_max(thr, NGX_QUIC_TIME_GRANULARITY); > @@ -179,21 +179,23 @@ ngx_quic_rtt_sample(ngx_connection_t *c, > enum ssl_encryption_level_t level, ngx_msec_t send_time) > { > ngx_msec_t latest_rtt, ack_delay, adjusted_rtt, rttvar_sample; > + ngx_quic_path_t *path; > ngx_quic_connection_t *qc; > > qc = ngx_quic_get_connection(c); > + path = qc->path; Technically this is not correct. The correct path for update is pkt->path from ngx_quic_handle_ack_frame() which may differ from qc->path. TBH accessing qc->path seems suspicious to me in most cases since pkt->path is always a better choice, except where there's no pkt in current context. However, even after fixing that, the result is still incorrect since we need to track rtt for the path the initial packet was sent over. It's also desirable for the ACK to be received on the same path IMHO. This is partially addressed by the next patch (not for rtt though, but for congestion). I suggest that we pass pkt->path to this function and update the next patch to address the remaining issues. > latest_rtt = ngx_current_msec - send_time; > - qc->latest_rtt = latest_rtt; > + path->latest_rtt = latest_rtt; > > - if (qc->min_rtt == NGX_TIMER_INFINITE) { > - qc->min_rtt = latest_rtt; > - qc->avg_rtt = latest_rtt; > - qc->rttvar = latest_rtt / 2; > - qc->first_rtt = ngx_current_msec; > + if (path->min_rtt == NGX_TIMER_INFINITE) { > + path->min_rtt = latest_rtt; > + path->avg_rtt = latest_rtt; > + path->rttvar = latest_rtt / 2; > + path->first_rtt = ngx_current_msec; > > } else { > - qc->min_rtt = ngx_min(qc->min_rtt, latest_rtt); > + path->min_rtt = ngx_min(path->min_rtt, latest_rtt); > > ack_delay = (ack->delay << qc->ctp.ack_delay_exponent) / 1000; > > @@ -203,18 +205,19 @@ ngx_quic_rtt_sample(ngx_connection_t *c, > > adjusted_rtt = latest_rtt; > > - if (qc->min_rtt + ack_delay < latest_rtt) { > + if (path->min_rtt + ack_delay < latest_rtt) { > adjusted_rtt -= ack_delay; > } > > - qc->avg_rtt += (adjusted_rtt >> 3) - (qc->avg_rtt >> 3); > - rttvar_sample = ngx_abs((ngx_msec_int_t) (qc->avg_rtt - adjusted_rtt)); > - qc->rttvar += (rttvar_sample >> 2) - (qc->rttvar >> 2); > + path->avg_rtt += (adjusted_rtt >> 3) - (path->avg_rtt >> 3); > + rttvar_sample = ngx_abs((ngx_msec_int_t) > + (path->avg_rtt - adjusted_rtt)); > + path->rttvar += (rttvar_sample >> 2) - (path->rttvar >> 2); > } > > ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, > "quic rtt sample latest:%M min:%M avg:%M var:%M", > - latest_rtt, qc->min_rtt, qc->avg_rtt, qc->rttvar); > + latest_rtt, path->min_rtt, path->avg_rtt, path->rttvar); > } > > > @@ -317,7 +320,7 @@ ngx_quic_congestion_ack(ngx_connection_t > } > > qc = ngx_quic_get_connection(c); > - cg = &qc->congestion; > + cg = &qc->path->congestion; We need to find the right path here, but the next patch fixes this. > blocked = (cg->in_flight >= cg->window) ? 1 : 0; > > @@ -428,13 +431,15 @@ ngx_quic_detect_lost(ngx_connection_t *c > ngx_uint_t i, nlost; > ngx_msec_t now, wait, thr, oldest, newest; > ngx_queue_t *q; > + ngx_quic_path_t *path; > ngx_quic_frame_t *start; > ngx_quic_send_ctx_t *ctx; > ngx_quic_connection_t *qc; > > qc = ngx_quic_get_connection(c); > + path = qc->path; Not sure what to do here, but it's obvious that we can have packets sent over different paths in ctx->sent with different "thr" and "first_rtt". Probably we need to find the right path for each packet in loop. This may delay followup packets which could otherwise be resent earlier though. Or just ignore all that and use qc->path, since everything else is so much harder and the effect from switching to qc->path for all packets may be just a small shift in waiting time before resend. > now = ngx_current_msec; > - thr = ngx_quic_lost_threshold(qc); > + thr = ngx_quic_lost_threshold(path); > > /* send time of lost packets across all send contexts */ > oldest = NGX_TIMER_INFINITE; > @@ -471,7 +476,7 @@ ngx_quic_detect_lost(ngx_connection_t *c > break; > } > > - if (start->last > qc->first_rtt) { > + if (start->last > path->first_rtt) { > > if (oldest == NGX_TIMER_INFINITE || start->last < oldest) { > oldest = start->last; > @@ -519,8 +524,8 @@ ngx_quic_pcg_duration(ngx_connection_t * > > qc = ngx_quic_get_connection(c); > > - duration = qc->avg_rtt; > - duration += ngx_max(4 * qc->rttvar, NGX_QUIC_TIME_GRANULARITY); > + duration = qc->path->avg_rtt; > + duration += ngx_max(4 * qc->path->rttvar, NGX_QUIC_TIME_GRANULARITY); > duration += qc->ctp.max_ack_delay; > duration *= NGX_QUIC_PERSISTENT_CONGESTION_THR; > > @@ -535,7 +540,7 @@ ngx_quic_persistent_congestion(ngx_conne > ngx_quic_connection_t *qc; > > qc = ngx_quic_get_connection(c); > - cg = &qc->congestion; > + cg = &qc->path->congestion; > > cg->recovery_start = ngx_current_msec; > cg->window = qc->tp.max_udp_payload_size * 2; > @@ -656,7 +661,7 @@ ngx_quic_congestion_lost(ngx_connection_ > } > > qc = ngx_quic_get_connection(c); > - cg = &qc->congestion; > + cg = &qc->path->congestion; We need to find the right path here for the frame. The third patch addresses this, but I suggest another solution, see below. > blocked = (cg->in_flight >= cg->window) ? 1 : 0; > > @@ -721,7 +726,8 @@ ngx_quic_set_lost_timer(ngx_connection_t > if (ctx->largest_ack != NGX_QUIC_UNSET_PN) { > q = ngx_queue_head(&ctx->sent); > f = ngx_queue_data(q, ngx_quic_frame_t, queue); > - w = (ngx_msec_int_t) (f->last + ngx_quic_lost_threshold(qc) - now); > + w = (ngx_msec_int_t) > + (f->last + ngx_quic_lost_threshold(qc->path) - now); > > if (f->pnum <= ctx->largest_ack) { > if (w < 0 || ctx->largest_ack - f->pnum >= NGX_QUIC_PKT_THR) { > @@ -777,17 +783,19 @@ ngx_msec_t > ngx_quic_pto(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) > { > ngx_msec_t duration; > + ngx_quic_path_t *path; > ngx_quic_connection_t *qc; > > qc = ngx_quic_get_connection(c); > + path = qc->path; > > /* RFC 9002, Appendix A.8. Setting the Loss Detection Timer */ > - duration = qc->avg_rtt; > + duration = path->avg_rtt; > > - duration += ngx_max(4 * qc->rttvar, NGX_QUIC_TIME_GRANULARITY); > + duration += ngx_max(4 * path->rttvar, NGX_QUIC_TIME_GRANULARITY); > duration <<= qc->pto_count; > > - if (qc->congestion.in_flight == 0) { /* no in-flight packets */ > + if (path->congestion.in_flight == 0) { /* no in-flight packets */ > return duration; > } > > diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h > --- a/src/event/quic/ngx_event_quic_connection.h > +++ b/src/event/quic/ngx_event_quic_connection.h > @@ -80,6 +80,14 @@ struct ngx_quic_server_id_s { > }; > > > +typedef struct { > + size_t in_flight; > + size_t window; > + size_t ssthresh; > + ngx_msec_t recovery_start; > +} ngx_quic_congestion_t; > + > + > struct ngx_quic_path_s { > ngx_queue_t queue; > struct sockaddr *sockaddr; > @@ -96,6 +104,15 @@ struct ngx_quic_path_s { > uint64_t seqnum; > ngx_str_t addr_text; > u_char text[NGX_SOCKADDR_STRLEN]; > + > + ngx_msec_t first_rtt; > + ngx_msec_t latest_rtt; > + ngx_msec_t avg_rtt; > + ngx_msec_t min_rtt; > + ngx_msec_t rttvar; > + > + ngx_quic_congestion_t congestion; > + > unsigned validated:1; > unsigned validating:1; > unsigned limited:1; > @@ -143,14 +160,6 @@ typedef struct { > } ngx_quic_streams_t; > > > -typedef struct { > - size_t in_flight; > - size_t window; > - size_t ssthresh; > - ngx_msec_t recovery_start; > -} ngx_quic_congestion_t; > - > - > /* > * RFC 9000, 12.3. Packet Numbers > * > @@ -218,12 +227,6 @@ struct ngx_quic_connection_s { > ngx_event_t path_validation; > ngx_msec_t last_cc; > > - ngx_msec_t first_rtt; > - ngx_msec_t latest_rtt; > - ngx_msec_t avg_rtt; > - ngx_msec_t min_rtt; > - ngx_msec_t rttvar; > - > ngx_uint_t pto_count; > > ngx_queue_t free_frames; > @@ -237,7 +240,6 @@ struct ngx_quic_connection_s { > #endif > > ngx_quic_streams_t streams; > - ngx_quic_congestion_t congestion; > > off_t received; > > diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c > --- a/src/event/quic/ngx_event_quic_migration.c > +++ b/src/event/quic/ngx_event_quic_migration.c > @@ -135,17 +135,26 @@ valid: > { > /* address did not change */ > rst = 0; > + > + path->avg_rtt = prev->avg_rtt; > + path->rttvar = prev->rttvar; > + path->min_rtt = prev->min_rtt; > + path->first_rtt = prev->first_rtt; > + path->latest_rtt = prev->latest_rtt; What if we introduce a new struct ngx_quic_rtt_t similar to ngx_quic_congestion_t? > + > + ngx_memcpy(&path->congestion, &prev->congestion, > + sizeof(ngx_quic_congestion_t)); > } > } > > if (rst) { > - ngx_memzero(&qc->congestion, sizeof(ngx_quic_congestion_t)); > + ngx_memzero(&path->congestion, sizeof(ngx_quic_congestion_t)); > > - qc->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size, > - ngx_max(2 * qc->tp.max_udp_payload_size, > - 14720)); > - qc->congestion.ssthresh = (size_t) -1; > - qc->congestion.recovery_start = ngx_current_msec; > + path->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size, > + ngx_max(2 * qc->tp.max_udp_payload_size, > + 14720)); > + path->congestion.ssthresh = (size_t) -1; > + path->congestion.recovery_start = ngx_current_msec; We need to reset rtt as well. On confirming a peer's ownership of its new address, an endpoint MUST immediately reset the congestion controller and **round-trip time** estimator for the new path to initial values We can do it in a separate patch. Also, I feel like we need separate functions for initializing rtt and congestion. The congestion initializer was present in the initial version of the patch as one of congestion methods. I think even though we have no methods now, the function still makes sense. > } > > /* > @@ -217,6 +226,21 @@ ngx_quic_new_path(ngx_connection_t *c, > path->addr_text.len = ngx_sock_ntop(sockaddr, socklen, path->text, > NGX_SOCKADDR_STRLEN, 1); > > + path->avg_rtt = NGX_QUIC_INITIAL_RTT; > + path->rttvar = NGX_QUIC_INITIAL_RTT / 2; > + path->min_rtt = NGX_TIMER_INFINITE; > + path->first_rtt = NGX_TIMER_INFINITE; > + > + /* > + * path->latest_rtt = 0 > + */ > + > + path->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size, > + ngx_max(2 * qc->tp.max_udp_payload_size, > + 14720)); > + path->congestion.ssthresh = (size_t) -1; > + path->congestion.recovery_start = ngx_current_msec; > + > ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, > "quic path seq:%uL created addr:%V", > path->seqnum, &path->addr_text); > diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c > --- a/src/event/quic/ngx_event_quic_output.c > +++ b/src/event/quic/ngx_event_quic_output.c > @@ -87,7 +87,7 @@ ngx_quic_output(ngx_connection_t *c) > c->log->action = "sending frames"; > > qc = ngx_quic_get_connection(c); > - cg = &qc->congestion; > + cg = &qc->path->congestion; > > in_flight = cg->in_flight; > > @@ -135,8 +135,8 @@ ngx_quic_create_datagrams(ngx_connection > static u_char dst[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE]; > > qc = ngx_quic_get_connection(c); > - cg = &qc->congestion; > path = qc->path; > + cg = &path->congestion; > > while (cg->in_flight < cg->window) { > > @@ -222,8 +222,7 @@ ngx_quic_commit_send(ngx_connection_t *c > ngx_quic_connection_t *qc; > > qc = ngx_quic_get_connection(c); > - > - cg = &qc->congestion; > + cg = &qc->path->congestion; > > while (!ngx_queue_empty(&ctx->sending)) { > > @@ -336,8 +335,8 @@ ngx_quic_create_segments(ngx_connection_ > static u_char dst[NGX_QUIC_MAX_UDP_SEGMENT_BUF]; > > qc = ngx_quic_get_connection(c); > - cg = &qc->congestion; > path = qc->path; > + cg = &path->congestion; > > ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); > > # HG changeset patch > # User Sergey Kandaurov > # Date 1672317973 -14400 > # Thu Dec 29 16:46:13 2022 +0400 > # Branch quic > # Node ID 0c8d81ada23c23f079db5c9c7f60d0cd3768555a > # Parent 46e738a5c0ab9d98577b21e72447cc9a6e3e9784 > QUIC: path aware in-flight bytes accounting. > > On packet acknowledgement is made path aware, as per RFC 9000, section 9.4: > Packets sent on the old path MUST NOT contribute to congestion control > or RTT estimation for the new path. > > Previously, the in-flight counter could be decremented for the wrong path. > If the active path was switched on connection migration with in-flight > contributing packets, the acknowledgement received after the congestion > controller is reset resulted in the counter underflow on the new path. > > diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c > --- a/src/event/quic/ngx_event_quic_ack.c > +++ b/src/event/quic/ngx_event_quic_ack.c > @@ -312,6 +312,8 @@ ngx_quic_congestion_ack(ngx_connection_t > { > ngx_uint_t blocked; > ngx_msec_t timer; > + ngx_queue_t *q; > + ngx_quic_path_t *path; > ngx_quic_congestion_t *cg; > ngx_quic_connection_t *qc; > > @@ -320,7 +322,27 @@ ngx_quic_congestion_ack(ngx_connection_t > } > > qc = ngx_quic_get_connection(c); > - cg = &qc->path->congestion; > + > +#if (NGX_SUPPRESS_WARN) > + path = NULL; > +#endif > + > + for (q = ngx_queue_head(&qc->paths); > + q != ngx_queue_sentinel(&qc->paths); > + q = ngx_queue_next(q)) > + { > + path = ngx_queue_data(q, ngx_quic_path_t, queue); > + > + if (path == f->path) { > + break; > + } > + } > + > + if (q == ngx_queue_sentinel(&qc->paths)) { > + return; > + } > + > + cg = &path->congestion; > > blocked = (cg->in_flight >= cg->window) ? 1 : 0; > > diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c > --- a/src/event/quic/ngx_event_quic_output.c > +++ b/src/event/quic/ngx_event_quic_output.c > @@ -234,6 +234,7 @@ ngx_quic_commit_send(ngx_connection_t *c > if (f->pkt_need_ack && !qc->closing) { > ngx_queue_insert_tail(&ctx->sent, q); > > + f->path = qc->path; > cg->in_flight += f->plen; > > } else { > diff --git a/src/event/quic/ngx_event_quic_transport.h b/src/event/quic/ngx_event_quic_transport.h > --- a/src/event/quic/ngx_event_quic_transport.h > +++ b/src/event/quic/ngx_event_quic_transport.h > @@ -265,6 +265,7 @@ struct ngx_quic_frame_s { > ngx_uint_t type; > enum ssl_encryption_level_t level; > ngx_queue_t queue; > + ngx_quic_path_t *path; This does not look reliable to me since paths may be deleted and then reused. I suggest that we use path->seqnum for this purpose. Currently it's only used for debugging. A similar change is needed in ngx_quic_congestion_lost(). > uint64_t pnum; > size_t plen; > ngx_msec_t first; > # HG changeset patch > # User Sergey Kandaurov > # Date 1672317976 -14400 > # Thu Dec 29 16:46:16 2022 +0400 > # Branch quic > # Node ID c450d58ba0108311c6e8dfa3e1ef15b267e12c9a > # Parent 0c8d81ada23c23f079db5c9c7f60d0cd3768555a > QUIC: avoid sending in-flight packets on a not yet validated path. > > Sending in-flight packets on a not yet validated path followed by confirming > a peer's ownership of its new address and the congestion controller reset, as > per RFC 9000, section 9.4, resulted in the lost accouting of in-flight bytes > and the bytes counter underflow on subsequent acknowledgement. > In practice, this occurred with NEW_CONNECTION_ID sent in response to peer's > RETIRE_CONNECTION_ID, which is acknowledged after the new path is validated. > > Since we change the address to send to in response to highest-numbered packets, > this measure should be sufficiently safe as an interim solution. IMHO it's better to track which packets contributed to in_flight to make sure we only subtract them. When confirming the ownership and resetting the congestion, we can increment path seqnum to make sure the old packets will not be subtracted later. > diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c > --- a/src/event/quic/ngx_event_quic_output.c > +++ b/src/event/quic/ngx_event_quic_output.c > @@ -153,6 +153,10 @@ ngx_quic_create_datagrams(ngx_connection > > ctx = &qc->send_ctx[i]; > > + if (ctx->level == ssl_encryption_application && !path->validated) { > + break; > + } > + > preserved_pnum[i] = ctx->pnum; > > if (ngx_quic_generate_ack(c, ctx) != NGX_OK) { > # HG changeset patch > # User Sergey Kandaurov > # Date 1672317981 -14400 > # Thu Dec 29 16:46:21 2022 +0400 > # Branch quic > # Node ID 1afa9e4c5dd7d2ff1b5f9b4fe521b429fc44e78c > # Parent c450d58ba0108311c6e8dfa3e1ef15b267e12c9a > QUIC: updating the local sockaddr when receiving a QUIC packet. > > In addition to saving the local sockaddr when establishing a new connection, > this change updates the connection local sockaddr on the next received packet. > The cached value will be used to set the property of a new path, which aims > to be different when using a preferred address. This and the following patches implement preferred address which I suggest we put on hold for now. First, my patch that enabled preferred address feature within a single listener is no longer considered for commit. Second, we should make sure the demand is there for a limited feature like this. Alternatively, we can implement full server context migration from a handshake QUIC server to a worker QUIC server, subject to demand. [..] -- Roman Arutyunyan From mdounin at mdounin.ru Mon Jan 23 02:57:40 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Mon, 23 Jan 2023 05:57:40 +0300 Subject: [PATCH] Configure: removed unneeded header from UDP_SEGMENT test Message-ID: # HG changeset patch # User Maxim Dounin # Date 1674439261 -10800 # Mon Jan 23 05:01:01 2023 +0300 # Node ID ec26874c3f60b115eea5a75b383aaa482837e628 # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 Configure: removed unneeded header from UDP_SEGMENT test. diff --git a/auto/os/linux b/auto/os/linux --- a/auto/os/linux +++ b/auto/os/linux @@ -238,7 +238,6 @@ ngx_feature="UDP_SEGMENT" ngx_feature_name="NGX_HAVE_UDP_SEGMENT" ngx_feature_run=no ngx_feature_incs="#include - #include #include " ngx_feature_path= ngx_feature_libs= From mdounin at mdounin.ru Mon Jan 23 02:59:17 2023 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Mon, 23 Jan 2023 05:59:17 +0300 Subject: [PATCH] QUIC: improved SO_COOKIE configure test Message-ID: <849f3b4043ee07a65bbf.1674442757@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1674441584 -10800 # Mon Jan 23 05:39:44 2023 +0300 # Branch quic # Node ID 849f3b4043ee07a65bbfc4ad136e4246002ec00c # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 QUIC: improved SO_COOKIE configure test. In nginx source code the inttypes.h include, if available, is used to define standard integer types. Changed the SO_COOKIE configure test to follow this. diff --git a/auto/os/linux b/auto/os/linux --- a/auto/os/linux +++ b/auto/os/linux @@ -263,7 +263,7 @@ ngx_feature="SO_COOKIE" ngx_feature_name="NGX_HAVE_SO_COOKIE" ngx_feature_run=no ngx_feature_incs="#include - #include " + $NGX_INCLUDE_INTTYPES_H" ngx_feature_path= ngx_feature_libs= ngx_feature_test="socklen_t optlen = sizeof(uint64_t); From mdounin at mdounin.ru Mon Jan 23 04:05:43 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 23 Jan 2023 07:05:43 +0300 Subject: [PATCH] HTTP: trigger lingering close when keepalive connection will be closed In-Reply-To: <699749D2-D96F-47D8-85DA-74E7F97682FF@gmail.com> References: <699749D2-D96F-47D8-85DA-74E7F97682FF@gmail.com> Message-ID: Hello! On Wed, Jan 18, 2023 at 11:28:52PM +0800, Miao Wang wrote: > # HG changeset patch > # User Miao Wang > # Date 1674055068 -28800 > # Wed Jan 18 23:17:48 2023 +0800 > # Node ID 73aa64bd29f3dec9e43e97560d6b5a07cdf40063 > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > HTTP: trigger lingering close when keepalive connection will be closed > > When finalizing a request, if the request is not keepalive but > its connection has served more than one request, then the connection > has been a keepalive connection previously and this connection will > be closed after this response. In this condition, it is likely that > there are pipelined requests following this request, which we should > ignore. As a result, lingering close is necessary in this case. > > Without this patch, nginx (with its default configuration) will send > out TCP RST when there are more pipelined requests. The symptom is > obvious when nginx is serving a debian repository and apt is > downloading massive of packages. See [1]. It becomes more obvious > when `keepalive_requests` is lower or nginx is under a relative > higher load, and it disappears when specifying > `lingering_close always`. > > [1]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 > > diff -r 07b0bee87f32 -r 73aa64bd29f3 src/http/ngx_http_request.c > --- a/src/http/ngx_http_request.c Wed Dec 21 14:53:27 2022 +0300 > +++ b/src/http/ngx_http_request.c Wed Jan 18 23:17:48 2023 +0800 > @@ -2749,6 +2749,10 @@ > return; > } > > + if (!r->keepalive && r->connection->requests > 1) { > + r->lingering_close = 1; > + } > + > if (clcf->lingering_close == NGX_HTTP_LINGERING_ALWAYS > || (clcf->lingering_close == NGX_HTTP_LINGERING_ON > && (r->lingering_close Thanks for the patch and the link to the Debian bug report. Lingering close implies noticeable additional resource usage: even if nothing happens on the connection, it will be kept open for lingering_timeout, which is 5 seconds by default. Given that pipelining is not used by most of the clients, forcing lingering close for all clients which are using keepalive does not look like a good solution to me. In general, nginx tries hard to determine if any additional data are expected on the connection, and uses lingering close if there is a good chance there will be some, but avoids lingering close by default if additional data are unlikely. If this logic does not work for some reason, lingering close can be explicitly requested with "lingering_close always;". In particular, note the "r->header_in->pos < r->header_in->last" and "r->connection->read->ready" checks - these are expected to catch connections with additional pipelined requests (see revision 3981:77604e9a1ed8). And from the log provided in the report it looks like it works most of the time - there are more than 6k HTTP requests, and 60+ connections. But sometimes it fails - there are two RST errors logged (and one "Undetermined Error", which looks like a bug in apt, but might be related). It looks like when apt is downloading many resources, it does not send all the requests at once (or in batches), but instead tries to maintain a constant "depth", a number of pipelined requests in flight. This essentially means that after reading of a response it sends an additional request. I see at least two possible cases which can result in nginx not using lingering close with such a load: 1. If a response where keepalive_requests is reached happens to be the last request in the r->header_in buffer (so the "r->header_in->pos < r->header_in->last" won't be effective), and there is a chance that nginx wasn't yet got an event from kernel about additional data (and therefore "r->connection->read->ready" will not be set). As such, nginx won't use lingering close, and might close connection with unread data in the socket buffer, resulting in RST. 2. Similarly, if nginx happens to be faster than apt, and socket buffers are large enough, it might sent all the responses, including the last one with "Connection: close", and close the connection (since there are no pending pipelined requests at the moment) even before an additional request is sent by apt. When later apt will send an additional request after reading some of the responses, it will send the request to already closed connection, again resulting in RST. It would be interesting to see more details, such as tcpdump and/or nginx debug logs, to find out what actually goes on here. Overall, given how apt uses pipelining, I tend to think that at least (2) is unavoidable and can happen with certain sizes of the responses. A good enough solution might be check for r->pipeline, which is set by nginx as long as it reads a pipelined request. It might not be enough though, since r->pipeline is only set for requests seen by nginx as pipelined, which might not be true for the last request. A more complete solution might be to introduce something like c->pipeline flag and use lingering close if any pipelined requests were seen on the connection. -- Maxim Dounin http://mdounin.ru/ From pluknet at nginx.com Mon Jan 23 10:21:33 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Mon, 23 Jan 2023 14:21:33 +0400 Subject: [PATCH] Added warning about redefinition of listen socket protocol options In-Reply-To: References: Message-ID: <5450B60D-DD89-4861-BA4D-6B57F085F7FD@nginx.com> > On 31 Dec 2022, at 18:35, Maxim Dounin wrote: > > # HG changeset patch > # User Maxim Dounin > # Date 1672497248 -10800 > # Sat Dec 31 17:34:08 2022 +0300 > # Node ID c215d5cf25732ece1819cf1cd48ebb480bb642c7 > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > Added warning about redefinition of listen socket protocol options. > > The "listen" directive in the http module can be used multiple times > in different server blocks. Originally, it was supposed to be specified > once with various socket options, and without any parameters in virtual > server blocks. For example: > > server { listen 80 backlog=1024; server_name foo; ... } > server { listen 80; server_name bar; ... } > server { listen 80; server_name bazz; ... } > > The address part of the syntax ("address[:port]" / "port" / "unix:path") > uniquely identifies the listening socket, and therefore is enough for > name-based virtual servers (to let nginx know that the virtual server > accepts requests on the listening socket in question). > > To ensure that listening options do not conflict between virtual servers, > they were allowed only once. For example, the following configuration > will be rejected ("duplicate listen options for 0.0.0.0:80 in ..."): > > server { listen 80 backlog=1024; server_name foo; ... } > server { listen 80 backlog=512; server_name bar; ... } > > At some point it was, however, noticed, that it is sometimes convenient > to repeat some options for clarity. In nginx 0.8.51 the "ssl" parameter > was allowed to be specified multiple times, e.g.: > > server { listen 443 ssl backlog=1024; server_name foo; ... } > server { listen 443 ssl; server_name bar; ... } > server { listen 443 ssl; server_name bazz; ... } > > This approach makes configuration more readable, since SSL sockets are > immediately visible in the configuration. If this is not needed, just the > address can still be used. > > Later, additional protocol-specific options similar to "ssl" were > introduced, notably "http2" and "proxy_protocol". With these options, > one can write: > > server { listen 443 ssl backlog=1024; server_name foo; ... } > server { listen 443 http2; server_name bar; ... } > server { listen 443 proxy_protocol; server_name bazz; ... } > > The resulting socket will use ssl, http2, and proxy_protocol, but this > is not really obvious from the configuration. > > To ensure such misleading configurations are not allowed, nginx now > warns as long as the "listen" directive is used with options different nitpicking: "ensure .. are not allowed" and "warns" don't seem to be equally strong. As such, I'd rewrite to something like: To emphasize such misleading configurations are discouraged, nginx now > from the options previously used if these are potentially confusing. Not really confident what "these" refers to. s/these/they/ ? > > In particular, the following configurations are allowed: > > server { listen 8401 ssl backlog=1024; server_name foo; } > server { listen 8401 ssl; server_name bar; } > server { listen 8401 ssl; server_name bazz; } > > server { listen 8402 ssl http2 backlog=1024; server_name foo; } > server { listen 8402 ssl; server_name bar; } > server { listen 8402 ssl; server_name bazz; } > > server { listen 8403 ssl; server_name bar; } > server { listen 8403 ssl; server_name bazz; } > server { listen 8403 ssl http2; server_name foo; } > > server { listen 8404 ssl http2 backlog=1024; server_name foo; } > server { listen 8404 http2; server_name bar; } > server { listen 8404 http2; server_name bazz; } > > server { listen 8405 ssl http2 backlog=1024; server_name foo; } > server { listen 8405 ssl http2; server_name bar; } > server { listen 8405 ssl http2; server_name bazz; } > > server { listen 8406 ssl; server_name foo; } > server { listen 8406; server_name bar; } > server { listen 8406; server_name bazz; } > > And the following configurations will generate warnings: > > server { listen 8501 ssl http2 backlog=1024; server_name foo; } > server { listen 8501 http2; server_name bar; } > server { listen 8501 ssl; server_name bazz; } > > server { listen 8502 backlog=1024; server_name foo; } > server { listen 8502 ssl; server_name bar; } > > server { listen 8503 ssl; server_name foo; } > server { listen 8503 http2; server_name bar; } > > server { listen 8504 ssl; server_name foo; } > server { listen 8504 http2; server_name bar; } > server { listen 8504 proxy_protocol; server_name bazz; } > > server { listen 8505 ssl http2 proxy_protocol; server_name foo; } > server { listen 8505 ssl http2; server_name bar; } > server { listen 8505 ssl; server_name bazz; } > > server { listen 8506 ssl http2; server_name foo; } > server { listen 8506 ssl; server_name bar; } > server { listen 8506; server_name bazz; } > > server { listen 8507 ssl; server_name bar; } > server { listen 8507; server_name bazz; } > server { listen 8507 ssl http2; server_name foo; } > > server { listen 8508 ssl; server_name bar; } > server { listen 8508; server_name bazz; } > server { listen 8508 ssl backlog=1024; server_name foo; } > > server { listen 8509; server_name bazz; } > server { listen 8509 ssl; server_name bar; } > server { listen 8509 ssl backlog=1024; server_name foo; } > 15 examples of dos and don'ts looks slightly excessive. The accurate description (such as provided by you below) allows to reduce most of them to e.g. four common invalid configurations: A lesser option set with socket option: server { listen 8443 backlog=1024; server_name foo; } server { listen 8443 http2; server_name bar; } The main option set is repeated at least twice: server { listen 127.0.0.1:8443; server_name foo; } server { listen 127.0.0.1:8443 ssl; server_name bar; } server { listen 127.0.0.1:8443 ssl; server_name baz; } Option sets partially overlap: server { listen 127.0.0.1:8443 ssl; server_name foo; } server { listen 127.0.0.1:8443 http2; server_name bar; } More than two option sets: server { listen 127.0.0.1:8443 http2 ssl; server_name foo; } server { listen 127.0.0.1:8443 http2; server_name bar; } server { listen 127.0.0.1:8443 ssl; server_name baz; } > The basic idea is that at most two sets of protocol options are allowed: > the main one (with socket options, if any), and a shorter one, with options > being a subset of the main options, repeated for clarity. As long as the > shorter set of protocol options is used, all listen directives except the > main one should use it. I'd move this paragraph somewhere before examples, as this is the most specific description of things actually changed. BTW, while reviewing I caught sort of a bug. As I understand the above explanation, if there are both full and short sets present, then at most one listen directive can have the full set, while shorter sets can be repeated. If so, then with the proposed patch the next configuration is expectedly invalid: server { listen 127.0.0.1:8443 ssl; server_name foo; } server { listen 127.0.0.1:8443 ssl; server_name bar; } server { listen 127.0.0.1:8443; server_name baz; } This is expected since first two servers with same options are interpreted as a short form (with full form seen potentially later on), but 3rd server has lesser options, which is caught by this check: (addr[i].protocols_set && protocols != addr[i].protocols) Which is interpreted as: "this server has a lesser set that doesn't match a a shorter set". Now, if 3rd server is moved first, configuration starts to pass: server { listen 127.0.0.1:8443; server_name baz; } server { listen 127.0.0.1:8443 ssl; server_name foo; } server { listen 127.0.0.1:8443 ssl; server_name bar; } This is because after (now) 2nd server, it is parsed as: 1st server has a short form, and 2nd server has a full form. Then 3rd server goes to "the same options" case. This also overwrites the remembered shorter set in addr[i].protocols. I guess an additional check should be added to address this. It is similar to the "options removed" case and ensures that the repeated options set actually matches a shorter set: diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c --- a/src/http/ngx_http.c +++ b/src/http/ngx_http.c @@ -1345,7 +1345,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, n /* the same options */ - if (lsopt->set && addr[i].protocols_changed) { + if ((lsopt->set && addr[i].protocols_changed) + || (addr[i].protocols_set && protocols != addr[i].protocols)) + { ngx_conf_log_error(NGX_LOG_WARN, cf, 0, "protocol options redefined for %V", &addr[i].opt.addr_text); > > diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c > --- a/src/http/ngx_http.c > +++ b/src/http/ngx_http.c > @@ -1228,7 +1228,8 @@ static ngx_int_t > ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, > ngx_http_conf_port_t *port, ngx_http_listen_opt_t *lsopt) > { > - ngx_uint_t i, default_server, proxy_protocol; > + ngx_uint_t i, default_server, proxy_protocol, > + protocols, protocols_prev; > ngx_http_conf_addr_t *addr; > #if (NGX_HTTP_SSL) > ngx_uint_t ssl; > @@ -1264,12 +1265,18 @@ ngx_http_add_addresses(ngx_conf_t *cf, n > default_server = addr[i].opt.default_server; > > proxy_protocol = lsopt->proxy_protocol || addr[i].opt.proxy_protocol; > + protocols = lsopt->proxy_protocol; > + protocols_prev = addr[i].opt.proxy_protocol; > > #if (NGX_HTTP_SSL) > ssl = lsopt->ssl || addr[i].opt.ssl; > + protocols |= lsopt->ssl << 1; > + protocols_prev |= addr[i].opt.ssl << 1; > #endif > #if (NGX_HTTP_V2) > http2 = lsopt->http2 || addr[i].opt.http2; > + protocols |= lsopt->http2 << 2; > + protocols_prev |= addr[i].opt.http2 << 2; > #endif > > if (lsopt->set) { > @@ -1299,6 +1306,55 @@ ngx_http_add_addresses(ngx_conf_t *cf, n > addr[i].default_server = cscf; > } > > + /* check for conflicting protocol options */ > + > + if ((protocols | protocols_prev) != protocols_prev) { > + > + /* options added */ > + > + if ((addr[i].opt.set && !lsopt->set) > + || addr[i].protocols_changed > + || (protocols | protocols_prev) != protocols) > + { > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > + "protocol options redefined for %V", > + &addr[i].opt.addr_text); > + } > + > + addr[i].protocols = protocols_prev; > + addr[i].protocols_set = 1; > + addr[i].protocols_changed = 1; > + > + } else if ((protocols_prev | protocols) != protocols) { > + > + /* options removed */ > + > + if (lsopt->set > + || (addr[i].protocols_set && protocols != addr[i].protocols)) > + { > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > + "protocol options redefined for %V", > + &addr[i].opt.addr_text); > + } > + > + addr[i].protocols = protocols; > + addr[i].protocols_set = 1; > + addr[i].protocols_changed = 1; > + > + } else { > + > + /* the same options */ > + > + if (lsopt->set && addr[i].protocols_changed) { > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > + "protocol options redefined for %V", > + &addr[i].opt.addr_text); > + } > + > + addr[i].protocols = protocols; > + addr[i].protocols_set = 1; > + } > + > addr[i].opt.default_server = default_server; > addr[i].opt.proxy_protocol = proxy_protocol; > #if (NGX_HTTP_SSL) > @@ -1355,6 +1411,9 @@ ngx_http_add_address(ngx_conf_t *cf, ngx > } > > addr->opt = *lsopt; > + addr->protocols = 0; > + addr->protocols_set = 0; > + addr->protocols_changed = 0; > addr->hash.buckets = NULL; > addr->hash.size = 0; > addr->wc_head = NULL; > diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h > --- a/src/http/ngx_http_core_module.h > +++ b/src/http/ngx_http_core_module.h > @@ -274,6 +274,10 @@ typedef struct { > typedef struct { > ngx_http_listen_opt_t opt; > > + unsigned protocols:3; > + unsigned protocols_set:1; > + unsigned protocols_changed:1; > + > ngx_hash_t hash; > ngx_hash_wildcard_t *wc_head; > ngx_hash_wildcard_t *wc_tail; -- Sergey Kandaurov From shankerwangmiao at gmail.com Mon Jan 23 11:01:16 2023 From: shankerwangmiao at gmail.com (Miao Wang) Date: Mon, 23 Jan 2023 19:01:16 +0800 Subject: [PATCH] HTTP: trigger lingering close when keepalive connection will be closed In-Reply-To: References: <699749D2-D96F-47D8-85DA-74E7F97682FF@gmail.com> Message-ID: <8D59E87C-2DC3-43AB-9067-019CA9ACE5C4@gmail.com> > 2023年1月23日 12:05，Maxim Dounin 写道： > > Hello! > > On Wed, Jan 18, 2023 at 11:28:52PM +0800, Miao Wang wrote: > >> # HG changeset patch >> # User Miao Wang >> # Date 1674055068 -28800 >> # Wed Jan 18 23:17:48 2023 +0800 >> # Node ID 73aa64bd29f3dec9e43e97560d6b5a07cdf40063 >> # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 >> HTTP: trigger lingering close when keepalive connection will be closed >> >> When finalizing a request, if the request is not keepalive but >> its connection has served more than one request, then the connection >> has been a keepalive connection previously and this connection will >> be closed after this response. In this condition, it is likely that >> there are pipelined requests following this request, which we should >> ignore. As a result, lingering close is necessary in this case. >> >> Without this patch, nginx (with its default configuration) will send >> out TCP RST when there are more pipelined requests. The symptom is >> obvious when nginx is serving a debian repository and apt is >> downloading massive of packages. See [1]. It becomes more obvious >> when `keepalive_requests` is lower or nginx is under a relative >> higher load, and it disappears when specifying >> `lingering_close always`. >> >> [1]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 >> >> diff -r 07b0bee87f32 -r 73aa64bd29f3 src/http/ngx_http_request.c >> --- a/src/http/ngx_http_request.c Wed Dec 21 14:53:27 2022 +0300 >> +++ b/src/http/ngx_http_request.c Wed Jan 18 23:17:48 2023 +0800 >> @@ -2749,6 +2749,10 @@ >> return; >> } >> >> + if (!r->keepalive && r->connection->requests > 1) { >> + r->lingering_close = 1; >> + } >> + >> if (clcf->lingering_close == NGX_HTTP_LINGERING_ALWAYS >> || (clcf->lingering_close == NGX_HTTP_LINGERING_ON >> && (r->lingering_close > > Thanks for the patch and the link to the Debian bug report. > > Lingering close implies noticeable additional resource usage: even > if nothing happens on the connection, it will be kept open for > lingering_timeout, which is 5 seconds by default. Given that > pipelining is not used by most of the clients, forcing lingering > close for all clients which are using keepalive does not look like > a good solution to me. > > In general, nginx tries hard to determine if any additional data > are expected on the connection, and uses lingering close if there > is a good chance there will be some, but avoids lingering close by > default if additional data are unlikely. If this logic does not > work for some reason, lingering close can be explicitly requested > with "lingering_close always;". That's true since the symptom I described can be worked around with that option. > > In particular, note the "r->header_in->pos < r->header_in->last" > and "r->connection->read->ready" checks - these are expected to > catch connections with additional pipelined requests (see revision > 3981:77604e9a1ed8). And from the log provided in the report it > looks like it works most of the time - there are more than 6k HTTP > requests, and 60+ connections. But sometimes it fails - there are > two RST errors logged (and one "Undetermined Error", which looks > like a bug in apt, but might be related). > > It looks like when apt is downloading many resources, it does not > send all the requests at once (or in batches), but instead tries > to maintain a constant "depth", a number of pipelined requests in > flight. This essentially means that after reading of a response > it sends an additional request. That's right. From a traffic dump, I can see apt first sends one request, and after receiving the response, it will send out 10 more requests, and maintain a depth of 10, since by default Acquire::http::Pipeline-Depth is 10. > > I see at least two possible cases which can result in nginx not > using lingering close with such a load: > > 1. If a response where keepalive_requests is reached happens to > be the last request in the r->header_in buffer (so the > "r->header_in->pos < r->header_in->last" won't be effective), and > there is a chance that nginx wasn't yet got an event from kernel > about additional data (and therefore "r->connection->read->ready" > will not be set). As such, nginx won't use lingering close, and > might close connection with unread data in the socket buffer, > resulting in RST. > > 2. Similarly, if nginx happens to be faster than apt, and socket > buffers are large enough, it might sent all the responses, > including the last one with "Connection: close", and close the > connection (since there are no pending pipelined requests at the > moment) even before an additional request is sent by apt. When > later apt will send an additional request after reading some of > the responses, it will send the request to already closed > connection, again resulting in RST. Actually, comparing the debug log and the pcap, nginx calls close() after writing the last response. However, at that time, that response is not fully transmitted to the client and there seems to be more requests not processed in the kernel buffer. Thus close() triggers an immediate RST. > > It would be interesting to see more details, such as tcpdump > and/or nginx debug logs, to find out what actually goes on here. The tcpdump and debug logs are too large to send in this mail list. I wonder if I can directly email it to you. > > Overall, given how apt uses pipelining, I tend to think that at > least (2) is unavoidable and can happen with certain sizes of the > responses. > > A good enough solution might be check for r->pipeline, which is > set by nginx as long as it reads a pipelined request. It might > not be enough though, since r->pipeline is only set for requests > seen by nginx as pipelined, which might not be true for the last > request. > > A more complete solution might be to introduce something like > c->pipeline flag and use lingering close if any pipelined requests > were seen on the connection. > > -- > Maxim Dounin > http://mdounin.ru/ > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel Many thanks for looking into this annoying issue. Cheers, Miao Wang From arut at nginx.com Mon Jan 23 12:28:50 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 23 Jan 2023 16:28:50 +0400 Subject: [PATCH] Configure: removed unneeded header from UDP_SEGMENT test In-Reply-To: References: Message-ID: <20230123122850.lirqtzlz5uiqucmg@N00W24XTQX> On Mon, Jan 23, 2023 at 05:57:40AM +0300, Maxim Dounin wrote: > # HG changeset patch > # User Maxim Dounin > # Date 1674439261 -10800 > # Mon Jan 23 05:01:01 2023 +0300 > # Node ID ec26874c3f60b115eea5a75b383aaa482837e628 > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > Configure: removed unneeded header from UDP_SEGMENT test. > > diff --git a/auto/os/linux b/auto/os/linux > --- a/auto/os/linux > +++ b/auto/os/linux > @@ -238,7 +238,6 @@ ngx_feature="UDP_SEGMENT" > ngx_feature_name="NGX_HAVE_UDP_SEGMENT" > ngx_feature_run=no > ngx_feature_incs="#include > - #include > #include " > ngx_feature_path= > ngx_feature_libs= > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel Looks ok From arut at nginx.com Mon Jan 23 12:30:01 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 23 Jan 2023 16:30:01 +0400 Subject: [PATCH] QUIC: improved SO_COOKIE configure test In-Reply-To: <849f3b4043ee07a65bbf.1674442757@vm-bsd.mdounin.ru> References: <849f3b4043ee07a65bbf.1674442757@vm-bsd.mdounin.ru> Message-ID: <20230123123001.scaslv6k5ap7dlos@N00W24XTQX> On Mon, Jan 23, 2023 at 05:59:17AM +0300, Maxim Dounin wrote: > # HG changeset patch > # User Maxim Dounin > # Date 1674441584 -10800 > # Mon Jan 23 05:39:44 2023 +0300 > # Branch quic > # Node ID 849f3b4043ee07a65bbfc4ad136e4246002ec00c > # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 > QUIC: improved SO_COOKIE configure test. > > In nginx source code the inttypes.h include, if available, is used to define > standard integer types. Changed the SO_COOKIE configure test to follow this. > > diff --git a/auto/os/linux b/auto/os/linux > --- a/auto/os/linux > +++ b/auto/os/linux > @@ -263,7 +263,7 @@ ngx_feature="SO_COOKIE" > ngx_feature_name="NGX_HAVE_SO_COOKIE" > ngx_feature_run=no > ngx_feature_incs="#include > - #include " > + $NGX_INCLUDE_INTTYPES_H" > ngx_feature_path= > ngx_feature_libs= > ngx_feature_test="socklen_t optlen = sizeof(uint64_t); > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel Looks ok From pluknet at nginx.com Mon Jan 23 17:28:42 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Mon, 23 Jan 2023 21:28:42 +0400 Subject: [PATCH] Gzip static: ranges support (ticket #2349) In-Reply-To: References: Message-ID: <262ABC58-7AB2-4B53-90D4-648464DE4F44@nginx.com> > On 3 Jan 2023, at 06:48, Maxim Dounin wrote: > > # HG changeset patch > # User Maxim Dounin > # Date 1672713976 -10800 > # Tue Jan 03 05:46:16 2023 +0300 > # Node ID e0688b4494f02dcf6feebf0c73e02749bd7de381 > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > Gzip static: ranges support (ticket #2349). > > In contrast to on-the-fly gzipping with gzip filter, static gzipped > representation as returned by gzip_static is persistent, and therefore > the same binary representation is available for future requests, making > it possible to use range requests. > > Further, if a gzipped representation is re-generated with different > compression settings, it is expected to result in different ETag and > different size reported in the Content-Range header, making it possible > to safely use range requests anyway. > > As such, ranges are now allowed for files returned by gzip_static. > > diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c > --- a/src/http/modules/ngx_http_gzip_static_module.c > +++ b/src/http/modules/ngx_http_gzip_static_module.c > @@ -247,6 +247,8 @@ ngx_http_gzip_static_handler(ngx_http_re > ngx_str_set(&h->value, "gzip"); > r->headers_out.content_encoding = h; > > + r->allow_ranges = 1; > + > /* we need to allocate all before the header would be sent */ > > b = ngx_calloc_buf(r->pool); Looks good. On a related note, while comparing with static module, which gzip_static is based on, I further noticed that gzip_static doesn't check for 0-size response in subrequests. Existing handling of r->main suggests that such configuration might be used in practice, e.g. together with gunzip filter, as documented in the gzip_static module documentation. So, it makes sense to add such check for zero size buffers as well. # HG changeset patch # User Sergey Kandaurov # Date 1674493925 -14400 # Mon Jan 23 21:12:05 2023 +0400 # Node ID 27217fca1966ddb20c843384d438df2af062fdfc # Parent dd458c69858b88231f542be4573a3f81141d1359 Gzip static: avoid "zero size buf" alerts in subrequests. Similar to the static module, gzip_static enabled in subrequests might result in zero size buffers with responses from empty precompressed files. diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c --- a/src/http/modules/ngx_http_gzip_static_module.c +++ b/src/http/modules/ngx_http_gzip_static_module.c @@ -236,6 +236,10 @@ ngx_http_gzip_static_handler(ngx_http_re return NGX_HTTP_INTERNAL_SERVER_ERROR; } + if (r != r->main && of.size == 0) { + return ngx_http_send_header(r); + } + h = ngx_list_push(&r->headers_out.headers); if (h == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; -- Sergey Kandaurov From mdounin at mdounin.ru Mon Jan 23 23:58:07 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 24 Jan 2023 02:58:07 +0300 Subject: [PATCH] Configure: removed unneeded header from UDP_SEGMENT test In-Reply-To: <20230123122850.lirqtzlz5uiqucmg@N00W24XTQX> References: <20230123122850.lirqtzlz5uiqucmg@N00W24XTQX> Message-ID: Hello! On Mon, Jan 23, 2023 at 04:28:50PM +0400, Roman Arutyunyan wrote: > On Mon, Jan 23, 2023 at 05:57:40AM +0300, Maxim Dounin wrote: > > # HG changeset patch > > # User Maxim Dounin > > # Date 1674439261 -10800 > > # Mon Jan 23 05:01:01 2023 +0300 > > # Node ID ec26874c3f60b115eea5a75b383aaa482837e628 > > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > > Configure: removed unneeded header from UDP_SEGMENT test. > > > > diff --git a/auto/os/linux b/auto/os/linux > > --- a/auto/os/linux > > +++ b/auto/os/linux > > @@ -238,7 +238,6 @@ ngx_feature="UDP_SEGMENT" > > ngx_feature_name="NGX_HAVE_UDP_SEGMENT" > > ngx_feature_run=no > > ngx_feature_incs="#include > > - #include > > #include " > > ngx_feature_path= > > ngx_feature_libs= > > _______________________________________________ > > nginx-devel mailing list > > nginx-devel at nginx.org > > https://mailman.nginx.org/mailman/listinfo/nginx-devel > > Looks ok Thanks for the review, pushed to http://mdounin.ru/hg/nginx. -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Mon Jan 23 23:59:02 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 24 Jan 2023 02:59:02 +0300 Subject: [PATCH] QUIC: improved SO_COOKIE configure test In-Reply-To: <20230123123001.scaslv6k5ap7dlos@N00W24XTQX> References: <849f3b4043ee07a65bbf.1674442757@vm-bsd.mdounin.ru> <20230123123001.scaslv6k5ap7dlos@N00W24XTQX> Message-ID: Hello! On Mon, Jan 23, 2023 at 04:30:01PM +0400, Roman Arutyunyan wrote: > On Mon, Jan 23, 2023 at 05:59:17AM +0300, Maxim Dounin wrote: > > # HG changeset patch > > # User Maxim Dounin > > # Date 1674441584 -10800 > > # Mon Jan 23 05:39:44 2023 +0300 > > # Branch quic > > # Node ID 849f3b4043ee07a65bbfc4ad136e4246002ec00c > > # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 > > QUIC: improved SO_COOKIE configure test. > > > > In nginx source code the inttypes.h include, if available, is used to define > > standard integer types. Changed the SO_COOKIE configure test to follow this. > > > > diff --git a/auto/os/linux b/auto/os/linux > > --- a/auto/os/linux > > +++ b/auto/os/linux > > @@ -263,7 +263,7 @@ ngx_feature="SO_COOKIE" > > ngx_feature_name="NGX_HAVE_SO_COOKIE" > > ngx_feature_run=no > > ngx_feature_incs="#include > > - #include " > > + $NGX_INCLUDE_INTTYPES_H" > > ngx_feature_path= > > ngx_feature_libs= > > ngx_feature_test="socklen_t optlen = sizeof(uint64_t); > > _______________________________________________ > > nginx-devel mailing list > > nginx-devel at nginx.org > > https://mailman.nginx.org/mailman/listinfo/nginx-devel > > Looks ok Thanks for the review, pushed to http://mdounin.ru/hg/nginx-quic. -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Tue Jan 24 02:19:03 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 24 Jan 2023 05:19:03 +0300 Subject: [PATCH] Gzip static: ranges support (ticket #2349) In-Reply-To: <262ABC58-7AB2-4B53-90D4-648464DE4F44@nginx.com> References: <262ABC58-7AB2-4B53-90D4-648464DE4F44@nginx.com> Message-ID: Hello! On Mon, Jan 23, 2023 at 09:28:42PM +0400, Sergey Kandaurov wrote: > > On 3 Jan 2023, at 06:48, Maxim Dounin wrote: > > > > # HG changeset patch > > # User Maxim Dounin > > # Date 1672713976 -10800 > > # Tue Jan 03 05:46:16 2023 +0300 > > # Node ID e0688b4494f02dcf6feebf0c73e02749bd7de381 > > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > > Gzip static: ranges support (ticket #2349). > > > > In contrast to on-the-fly gzipping with gzip filter, static gzipped > > representation as returned by gzip_static is persistent, and therefore > > the same binary representation is available for future requests, making > > it possible to use range requests. > > > > Further, if a gzipped representation is re-generated with different > > compression settings, it is expected to result in different ETag and > > different size reported in the Content-Range header, making it possible > > to safely use range requests anyway. > > > > As such, ranges are now allowed for files returned by gzip_static. > > > > diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c > > --- a/src/http/modules/ngx_http_gzip_static_module.c > > +++ b/src/http/modules/ngx_http_gzip_static_module.c > > @@ -247,6 +247,8 @@ ngx_http_gzip_static_handler(ngx_http_re > > ngx_str_set(&h->value, "gzip"); > > r->headers_out.content_encoding = h; > > > > + r->allow_ranges = 1; > > + > > /* we need to allocate all before the header would be sent */ > > > > b = ngx_calloc_buf(r->pool); > > Looks good. Thanks for the review, pushed to http://mdounin.ru/hg/nginx. > On a related note, while comparing with static module, which gzip_static > is based on, I further noticed that gzip_static doesn't check for 0-size > response in subrequests. Existing handling of r->main suggests that > such configuration might be used in practice, e.g. together with gunzip > filter, as documented in the gzip_static module documentation. > So, it makes sense to add such check for zero size buffers as well. > > # HG changeset patch > # User Sergey Kandaurov > # Date 1674493925 -14400 > # Mon Jan 23 21:12:05 2023 +0400 > # Node ID 27217fca1966ddb20c843384d438df2af062fdfc > # Parent dd458c69858b88231f542be4573a3f81141d1359 > Gzip static: avoid "zero size buf" alerts in subrequests. > > Similar to the static module, gzip_static enabled in subrequests might > result in zero size buffers with responses from empty precompressed files. This looks like an issue introduced in 4611:2b6cb7528409, and it might be a good idea to fix other affected modules as well. While valid gzipped responses are not expected to be empty, much like valid mp4 and flv files, but it certainly shouldn't cause "zero size buf" alerts if there is an invalid file for some reason. > diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c > --- a/src/http/modules/ngx_http_gzip_static_module.c > +++ b/src/http/modules/ngx_http_gzip_static_module.c > @@ -236,6 +236,10 @@ ngx_http_gzip_static_handler(ngx_http_re > return NGX_HTTP_INTERNAL_SERVER_ERROR; > } > > + if (r != r->main && of.size == 0) { > + return ngx_http_send_header(r); > + } > + > h = ngx_list_push(&r->headers_out.headers); > if (h == NULL) { > return NGX_HTTP_INTERNAL_SERVER_ERROR; While there seems to be no practical difference, I don't think that not adding the "Content-Encoding" header for empty subrequest responses is a good idea. Further, I don't actually think that skipping sending the body, as the static module currently does, is a good idea either. It might break various filter modules which might not expect such behaviour. I would rather consider sending the body as usual, but with b->sync set when sending an empty body in subrequests, similarly to how ngx_http_send_special() does. Something like this will do the trick: diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c --- a/src/http/modules/ngx_http_gzip_static_module.c +++ b/src/http/modules/ngx_http_gzip_static_module.c @@ -273,6 +273,7 @@ ngx_http_gzip_static_handler(ngx_http_re b->in_file = b->file_last ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (r == r->main || b->file_last) ? 0 : 1; b->file->fd = of.fd; b->file->name = path; In particular, with this patch inflate() is able to properly report errors on empty files: 2023/01/24 03:54:44 [error] 22886#100160: *1 inflate() returned -5 on response end while sending response to client, client: 127.0.0.1, server: , request: "GET /index.html HTTP/1.1", subrequest: "/empty.html", host: "127.0.0.1:8080" I also observe a similar behaviour change with empty xml files returned in subrequests by static module and processed by the xslt filter: previously, empty xml files were effectively ignored, and now they result in the errors much like any other malformed xml files. Full patch: # HG changeset patch # User Maxim Dounin # Date 1674526244 -10800 # Tue Jan 24 05:10:44 2023 +0300 # Node ID 22e41eed77ad95f51d7d0a3daa5cb369c9643697 # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db Fixed "zero size buf" alerts with subrequests. Since 4611:2b6cb7528409 responses from the gzip static, flv, and mp4 modules can be used with subrequests, though empty files were not properly handled. Empty gzipped, flv, and mp4 files thus resulted in "zero size buf in output" alerts. While valid corresponding files are not expected to be empty, such files shouldn't result in alerts. Fix is to set b->sync on such empty subrequest responses, similarly to what ngx_http_send_special() does. Additionally, the static module is modified to do the same instead of not sending the response body at all in such cases, since not sending the response body at all is believed to be at least questionable, and might break various filters which do not expect such behaviour. diff --git a/src/http/modules/ngx_http_flv_module.c b/src/http/modules/ngx_http_flv_module.c --- a/src/http/modules/ngx_http_flv_module.c +++ b/src/http/modules/ngx_http_flv_module.c @@ -235,6 +235,7 @@ ngx_http_flv_handler(ngx_http_request_t b->in_file = b->file_last ? 1: 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (r == r->main || b->file_last) ? 0 : 1; b->file->fd = of.fd; b->file->name = path; diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c --- a/src/http/modules/ngx_http_gzip_static_module.c +++ b/src/http/modules/ngx_http_gzip_static_module.c @@ -273,6 +273,7 @@ ngx_http_gzip_static_handler(ngx_http_re b->in_file = b->file_last ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (r == r->main || b->file_last) ? 0 : 1; b->file->fd = of.fd; b->file->name = path; diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c --- a/src/http/modules/ngx_http_mp4_module.c +++ b/src/http/modules/ngx_http_mp4_module.c @@ -714,6 +714,7 @@ ngx_http_mp4_handler(ngx_http_request_t b->in_file = b->file_last ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (r == r->main || b->file_last) ? 0 : 1; b->file->fd = of.fd; b->file->name = path; diff --git a/src/http/modules/ngx_http_static_module.c b/src/http/modules/ngx_http_static_module.c --- a/src/http/modules/ngx_http_static_module.c +++ b/src/http/modules/ngx_http_static_module.c @@ -238,10 +238,6 @@ ngx_http_static_handler(ngx_http_request return NGX_HTTP_INTERNAL_SERVER_ERROR; } - if (r != r->main && of.size == 0) { - return ngx_http_send_header(r); - } - r->allow_ranges = 1; /* we need to allocate all before the header would be sent */ @@ -268,6 +264,7 @@ ngx_http_static_handler(ngx_http_request b->in_file = b->file_last ? 1: 0; b->last_buf = (r == r->main) ? 1: 0; b->last_in_chain = 1; + b->sync = (r == r->main || b->file_last) ? 0 : 1; b->file->fd = of.fd; b->file->name = path; -- Maxim Dounin http://mdounin.ru/ From pluknet at nginx.com Tue Jan 24 11:16:49 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Tue, 24 Jan 2023 11:16:49 +0000 Subject: [nginx] Configure: removed unneeded header from UDP_SEGMENT test. Message-ID: details: https://hg.nginx.org/nginx/rev/ec26874c3f60 branches: changeset: 8119:ec26874c3f60 user: Maxim Dounin date: Mon Jan 23 05:01:01 2023 +0300 description: Configure: removed unneeded header from UDP_SEGMENT test. diffstat: auto/os/linux | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 07b0bee87f32 -r ec26874c3f60 auto/os/linux --- a/auto/os/linux Wed Dec 21 14:53:27 2022 +0300 +++ b/auto/os/linux Mon Jan 23 05:01:01 2023 +0300 @@ -238,7 +238,6 @@ ngx_feature="UDP_SEGMENT" ngx_feature_name="NGX_HAVE_UDP_SEGMENT" ngx_feature_run=no ngx_feature_incs="#include - #include #include " ngx_feature_path= ngx_feature_libs= From pluknet at nginx.com Tue Jan 24 11:16:52 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Tue, 24 Jan 2023 11:16:52 +0000 Subject: [nginx] Gzip static: ranges support (ticket #2349). Message-ID: details: https://hg.nginx.org/nginx/rev/c7e103acb409 branches: changeset: 8120:c7e103acb409 user: Maxim Dounin date: Tue Jan 24 03:01:51 2023 +0300 description: Gzip static: ranges support (ticket #2349). In contrast to on-the-fly gzipping with gzip filter, static gzipped representation as returned by gzip_static is persistent, and therefore the same binary representation is available for future requests, making it possible to use range requests. Further, if a gzipped representation is re-generated with different compression settings, it is expected to result in different ETag and different size reported in the Content-Range header, making it possible to safely use range requests anyway. As such, ranges are now allowed for files returned by gzip_static. diffstat: src/http/modules/ngx_http_gzip_static_module.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r ec26874c3f60 -r c7e103acb409 src/http/modules/ngx_http_gzip_static_module.c --- a/src/http/modules/ngx_http_gzip_static_module.c Mon Jan 23 05:01:01 2023 +0300 +++ b/src/http/modules/ngx_http_gzip_static_module.c Tue Jan 24 03:01:51 2023 +0300 @@ -247,6 +247,8 @@ ngx_http_gzip_static_handler(ngx_http_re ngx_str_set(&h->value, "gzip"); r->headers_out.content_encoding = h; + r->allow_ranges = 1; + /* we need to allocate all before the header would be sent */ b = ngx_calloc_buf(r->pool); From alx.manpages at gmail.com Tue Jan 24 23:22:54 2023 From: alx.manpages at gmail.com (Alex Colomar) Date: Wed, 25 Jan 2023 00:22:54 +0100 Subject: [PATCH] QUIC: improved SO_COOKIE configure test In-Reply-To: <849f3b4043ee07a65bbf.1674442757@vm-bsd.mdounin.ru> References: <849f3b4043ee07a65bbf.1674442757@vm-bsd.mdounin.ru> Message-ID: <4a8c5d37-f1ff-b9ab-f47c-33a155b76480@gmail.com> Hi Maxim, On 1/23/23 03:59, Maxim Dounin wrote: > # HG changeset patch > # User Maxim Dounin > # Date 1674441584 -10800 > # Mon Jan 23 05:39:44 2023 +0300 > # Branch quic > # Node ID 849f3b4043ee07a65bbfc4ad136e4246002ec00c > # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 > QUIC: improved SO_COOKIE configure test. > > In nginx source code the inttypes.h include, if available, is used to define > standard integer types. Changed the SO_COOKIE configure test to follow this. May I ask for curiosity why do you prefer for the standard integer types? Normally, the preferred header file is , since it's smaller; unless you use the macros or functions from , which are things like PRId32, or imaxabs(3). Is there any portability issue with ? Thanks, Alex > > diff --git a/auto/os/linux b/auto/os/linux > --- a/auto/os/linux > +++ b/auto/os/linux > @@ -263,7 +263,7 @@ ngx_feature="SO_COOKIE" > ngx_feature_name="NGX_HAVE_SO_COOKIE" > ngx_feature_run=no > ngx_feature_incs="#include > - #include " > + $NGX_INCLUDE_INTTYPES_H" > ngx_feature_path= > ngx_feature_libs= > ngx_feature_test="socklen_t optlen = sizeof(uint64_t); > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel -- -------------- next part -------------- A non-text attachment was scrubbed... Name: OpenPGP_signature Type: application/pgp-signature Size: 833 bytes Desc: OpenPGP digital signature URL: From mdounin at mdounin.ru Wed Jan 25 02:17:30 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Wed, 25 Jan 2023 05:17:30 +0300 Subject: [PATCH] HTTP: trigger lingering close when keepalive connection will be closed In-Reply-To: <8D59E87C-2DC3-43AB-9067-019CA9ACE5C4@gmail.com> References: <699749D2-D96F-47D8-85DA-74E7F97682FF@gmail.com> <8D59E87C-2DC3-43AB-9067-019CA9ACE5C4@gmail.com> Message-ID: Hello! On Mon, Jan 23, 2023 at 07:01:16PM +0800, Miao Wang wrote: > > 2023年1月23日 12:05，Maxim Dounin 写道： > > > > On Wed, Jan 18, 2023 at 11:28:52PM +0800, Miao Wang wrote: > > > >> # HG changeset patch > >> # User Miao Wang > >> # Date 1674055068 -28800 > >> # Wed Jan 18 23:17:48 2023 +0800 > >> # Node ID 73aa64bd29f3dec9e43e97560d6b5a07cdf40063 > >> # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > >> HTTP: trigger lingering close when keepalive connection will be closed > >> > >> When finalizing a request, if the request is not keepalive but > >> its connection has served more than one request, then the connection > >> has been a keepalive connection previously and this connection will > >> be closed after this response. In this condition, it is likely that > >> there are pipelined requests following this request, which we should > >> ignore. As a result, lingering close is necessary in this case. > >> > >> Without this patch, nginx (with its default configuration) will send > >> out TCP RST when there are more pipelined requests. The symptom is > >> obvious when nginx is serving a debian repository and apt is > >> downloading massive of packages. See [1]. It becomes more obvious > >> when `keepalive_requests` is lower or nginx is under a relative > >> higher load, and it disappears when specifying > >> `lingering_close always`. > >> > >> [1]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 > >> > >> diff -r 07b0bee87f32 -r 73aa64bd29f3 src/http/ngx_http_request.c > >> --- a/src/http/ngx_http_request.c Wed Dec 21 14:53:27 2022 +0300 > >> +++ b/src/http/ngx_http_request.c Wed Jan 18 23:17:48 2023 +0800 > >> @@ -2749,6 +2749,10 @@ > >> return; > >> } > >> > >> + if (!r->keepalive && r->connection->requests > 1) { > >> + r->lingering_close = 1; > >> + } > >> + > >> if (clcf->lingering_close == NGX_HTTP_LINGERING_ALWAYS > >> || (clcf->lingering_close == NGX_HTTP_LINGERING_ON > >> && (r->lingering_close > > > > Thanks for the patch and the link to the Debian bug report. > > > > Lingering close implies noticeable additional resource usage: even > > if nothing happens on the connection, it will be kept open for > > lingering_timeout, which is 5 seconds by default. Given that > > pipelining is not used by most of the clients, forcing lingering > > close for all clients which are using keepalive does not look like > > a good solution to me. > > > > In general, nginx tries hard to determine if any additional data > > are expected on the connection, and uses lingering close if there > > is a good chance there will be some, but avoids lingering close by > > default if additional data are unlikely. If this logic does not > > work for some reason, lingering close can be explicitly requested > > with "lingering_close always;". > > That's true since the symptom I described can be worked around with > that option. > > > > > In particular, note the "r->header_in->pos < r->header_in->last" > > and "r->connection->read->ready" checks - these are expected to > > catch connections with additional pipelined requests (see revision > > 3981:77604e9a1ed8). And from the log provided in the report it > > looks like it works most of the time - there are more than 6k HTTP > > requests, and 60+ connections. But sometimes it fails - there are > > two RST errors logged (and one "Undetermined Error", which looks > > like a bug in apt, but might be related). > > > > It looks like when apt is downloading many resources, it does not > > send all the requests at once (or in batches), but instead tries > > to maintain a constant "depth", a number of pipelined requests in > > flight. This essentially means that after reading of a response > > it sends an additional request. > > That's right. From a traffic dump, I can see apt first sends one > request, and after receiving the response, it will send out 10 > more requests, and maintain a depth of 10, since by default > Acquire::http::Pipeline-Depth is 10. > > > > > I see at least two possible cases which can result in nginx not > > using lingering close with such a load: > > > > 1. If a response where keepalive_requests is reached happens to > > be the last request in the r->header_in buffer (so the > > "r->header_in->pos < r->header_in->last" won't be effective), and > > there is a chance that nginx wasn't yet got an event from kernel > > about additional data (and therefore "r->connection->read->ready" > > will not be set). As such, nginx won't use lingering close, and > > might close connection with unread data in the socket buffer, > > resulting in RST. > > > > 2. Similarly, if nginx happens to be faster than apt, and socket > > buffers are large enough, it might sent all the responses, > > including the last one with "Connection: close", and close the > > connection (since there are no pending pipelined requests at the > > moment) even before an additional request is sent by apt. When > > later apt will send an additional request after reading some of > > the responses, it will send the request to already closed > > connection, again resulting in RST. > > Actually, comparing the debug log and the pcap, nginx calls > close() after writing the last response. However, at that time, > that response is not fully transmitted to the client and there > seems to be more requests not processed in the kernel buffer. > Thus close() triggers an immediate RST. Thanks for the details. This looks more like the first case, and probably can be addressed by improving likelihood of detecting the read event. Could you please test if the patch below fixes the particular issue you are seeing? It is somewhat unrelated, but it might be a good enough solution (and is more or less equivalent to checking r->pipeline). > > It would be interesting to see more details, such as tcpdump > > and/or nginx debug logs, to find out what actually goes on here. > > The tcpdump and debug logs are too large to send in this mail list. > I wonder if I can directly email it to you. Feel free to, my email should accept up to 100M messages. Alternatively, a good solution might be to make the files available for download and post a link here. > > Overall, given how apt uses pipelining, I tend to think that at > > least (2) is unavoidable and can happen with certain sizes of the > > responses. > > > > A good enough solution might be check for r->pipeline, which is > > set by nginx as long as it reads a pipelined request. It might > > not be enough though, since r->pipeline is only set for requests > > seen by nginx as pipelined, which might not be true for the last > > request. > > > > A more complete solution might be to introduce something like > > c->pipeline flag and use lingering close if any pipelined requests > > were seen on the connection. The following patch reworks handling of pipelined requests by postponing them to the next event loop iteration. It is expected make it more likely for nginx to know there are any additional unread data in the socket buffer (and right now is mostly equivalent to checking r->pipeline, since c->read->ready is always set for pipelined requests): # HG changeset patch # User Maxim Dounin # Date 1674610218 -10800 # Wed Jan 25 04:30:18 2023 +0300 # Node ID 8cfd22c325a3db370b9e45aa6f897ff7bc8222f3 # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db Reworked pipelined requests to use posted next events. This is expected to improve handling of pipelined requests in a number of ways, including: 1) It will make a room for additional requests from other clients, reducing worker monopolization by a single client. 2) The c->read->ready flag will be set, so nginx will either read the additional data, or will use lingering close. This is expected to help with clients using pipelining with some constant depth, such as apt[1][2]. The ngx_event_move_posted_next() was modified to make it possible to post read events on connections with kqueue. Previously, it used to set ev->available to -1, potentially overwriting a valid positive value provided by kqueue, so ngx_unix_recv() and ngx_readv_chain() will stop reading from the socket before reading all the data available. Note that currently ngx_event_move_posted_next() will always set the ev->ready flag. While this is expected behaviour for the ev->available use case (where ev->ready is explicitly cleared), this is not needed for pipelining. For pipelining, this will result in extra unneeded read() syscall after processing of all pipelined requests, and there might be a room for improvement here. [1] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 [2] https://mailman.nginx.org/pipermail/nginx-devel/2023-January/ZA2SP5SJU55LHEBCJMFDB2AZVELRLTHI.html diff --git a/src/event/ngx_event_posted.c b/src/event/ngx_event_posted.c --- a/src/event/ngx_event_posted.c +++ b/src/event/ngx_event_posted.c @@ -51,8 +51,10 @@ ngx_event_move_posted_next(ngx_cycle_t * ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "posted next event %p", ev); - ev->ready = 1; - ev->available = -1; + if (!ev->ready) { + ev->ready = 1; + ev->available = -1; + } } ngx_queue_add(&ngx_posted_events, &ngx_posted_next_events); diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -3129,7 +3129,7 @@ ngx_http_set_keepalive(ngx_http_request_ } rev->handler = ngx_http_process_request_line; - ngx_post_event(rev, &ngx_posted_events); + ngx_post_event(rev, &ngx_posted_next_events); return; } -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Wed Jan 25 03:19:29 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Wed, 25 Jan 2023 06:19:29 +0300 Subject: [PATCH] QUIC: improved SO_COOKIE configure test In-Reply-To: <4a8c5d37-f1ff-b9ab-f47c-33a155b76480@gmail.com> References: <849f3b4043ee07a65bbf.1674442757@vm-bsd.mdounin.ru> <4a8c5d37-f1ff-b9ab-f47c-33a155b76480@gmail.com> Message-ID: Hello! On Wed, Jan 25, 2023 at 12:22:54AM +0100, Alex Colomar wrote: > On 1/23/23 03:59, Maxim Dounin wrote: > > # HG changeset patch > > # User Maxim Dounin > > # Date 1674441584 -10800 > > # Mon Jan 23 05:39:44 2023 +0300 > > # Branch quic > > # Node ID 849f3b4043ee07a65bbfc4ad136e4246002ec00c > > # Parent 6bb884dc72916dc675df65d02abee0c9cfabc916 > > QUIC: improved SO_COOKIE configure test. > > > > In nginx source code the inttypes.h include, if available, is used to define > > standard integer types. Changed the SO_COOKIE configure test to follow this. > > May I ask for curiosity why do you prefer for the standard > integer types? > > Normally, the preferred header file is , since it's smaller; > unless you use the macros or functions from , which are > things like PRId32, or imaxabs(3). > > Is there any portability issue with ? The inttypes.h is more portable: for example, stdint.h is not available on FreeBSD before FreeBSD 5.0: https://github.com/freebsd/freebsd-src/commit/0ac2d551f20a8769869f61ebfe742fd55cef70b9 While this is not important for the particular test (since all known Linux versions with SO_COOKIE do have stdint.h), using the same header in all tests and the code is good from consistency point of view. -- Maxim Dounin http://mdounin.ru/ From shankerwangmiao at gmail.com Wed Jan 25 04:49:10 2023 From: shankerwangmiao at gmail.com (Miao Wang) Date: Wed, 25 Jan 2023 12:49:10 +0800 Subject: [PATCH] HTTP: trigger lingering close when keepalive connection will be closed In-Reply-To: References: <699749D2-D96F-47D8-85DA-74E7F97682FF@gmail.com> <8D59E87C-2DC3-43AB-9067-019CA9ACE5C4@gmail.com> Message-ID: Hi, > 2023年1月25日 10:17，Maxim Dounin 写道： > > Hello! > > On Mon, Jan 23, 2023 at 07:01:16PM +0800, Miao Wang wrote: > >>> 2023年1月23日 12:05，Maxim Dounin 写道： >>> >>> On Wed, Jan 18, 2023 at 11:28:52PM +0800, Miao Wang wrote: >>> >>>> # HG changeset patch >>>> # User Miao Wang >>>> # Date 1674055068 -28800 >>>> # Wed Jan 18 23:17:48 2023 +0800 >>>> # Node ID 73aa64bd29f3dec9e43e97560d6b5a07cdf40063 >>>> # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 >>>> HTTP: trigger lingering close when keepalive connection will be closed >>>> >>>> When finalizing a request, if the request is not keepalive but >>>> its connection has served more than one request, then the connection >>>> has been a keepalive connection previously and this connection will >>>> be closed after this response. In this condition, it is likely that >>>> there are pipelined requests following this request, which we should >>>> ignore. As a result, lingering close is necessary in this case. >>>> >>>> Without this patch, nginx (with its default configuration) will send >>>> out TCP RST when there are more pipelined requests. The symptom is >>>> obvious when nginx is serving a debian repository and apt is >>>> downloading massive of packages. See [1]. It becomes more obvious >>>> when `keepalive_requests` is lower or nginx is under a relative >>>> higher load, and it disappears when specifying >>>> `lingering_close always`. >>>> >>>> [1]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 >>>> >>>> diff -r 07b0bee87f32 -r 73aa64bd29f3 src/http/ngx_http_request.c >>>> --- a/src/http/ngx_http_request.c Wed Dec 21 14:53:27 2022 +0300 >>>> +++ b/src/http/ngx_http_request.c Wed Jan 18 23:17:48 2023 +0800 >>>> @@ -2749,6 +2749,10 @@ >>>> return; >>>> } >>>> >>>> + if (!r->keepalive && r->connection->requests > 1) { >>>> + r->lingering_close = 1; >>>> + } >>>> + >>>> if (clcf->lingering_close == NGX_HTTP_LINGERING_ALWAYS >>>> || (clcf->lingering_close == NGX_HTTP_LINGERING_ON >>>> && (r->lingering_close >>> >>> Thanks for the patch and the link to the Debian bug report. >>> >>> Lingering close implies noticeable additional resource usage: even >>> if nothing happens on the connection, it will be kept open for >>> lingering_timeout, which is 5 seconds by default. Given that >>> pipelining is not used by most of the clients, forcing lingering >>> close for all clients which are using keepalive does not look like >>> a good solution to me. >>> >>> In general, nginx tries hard to determine if any additional data >>> are expected on the connection, and uses lingering close if there >>> is a good chance there will be some, but avoids lingering close by >>> default if additional data are unlikely. If this logic does not >>> work for some reason, lingering close can be explicitly requested >>> with "lingering_close always;". >> >> That's true since the symptom I described can be worked around with >> that option. >> >>> >>> In particular, note the "r->header_in->pos < r->header_in->last" >>> and "r->connection->read->ready" checks - these are expected to >>> catch connections with additional pipelined requests (see revision >>> 3981:77604e9a1ed8). And from the log provided in the report it >>> looks like it works most of the time - there are more than 6k HTTP >>> requests, and 60+ connections. But sometimes it fails - there are >>> two RST errors logged (and one "Undetermined Error", which looks >>> like a bug in apt, but might be related). >>> >>> It looks like when apt is downloading many resources, it does not >>> send all the requests at once (or in batches), but instead tries >>> to maintain a constant "depth", a number of pipelined requests in >>> flight. This essentially means that after reading of a response >>> it sends an additional request. >> >> That's right. From a traffic dump, I can see apt first sends one >> request, and after receiving the response, it will send out 10 >> more requests, and maintain a depth of 10, since by default >> Acquire::http::Pipeline-Depth is 10. >> >>> >>> I see at least two possible cases which can result in nginx not >>> using lingering close with such a load: >>> >>> 1. If a response where keepalive_requests is reached happens to >>> be the last request in the r->header_in buffer (so the >>> "r->header_in->pos < r->header_in->last" won't be effective), and >>> there is a chance that nginx wasn't yet got an event from kernel >>> about additional data (and therefore "r->connection->read->ready" >>> will not be set). As such, nginx won't use lingering close, and >>> might close connection with unread data in the socket buffer, >>> resulting in RST. >>> >>> 2. Similarly, if nginx happens to be faster than apt, and socket >>> buffers are large enough, it might sent all the responses, >>> including the last one with "Connection: close", and close the >>> connection (since there are no pending pipelined requests at the >>> moment) even before an additional request is sent by apt. When >>> later apt will send an additional request after reading some of >>> the responses, it will send the request to already closed >>> connection, again resulting in RST. >> >> Actually, comparing the debug log and the pcap, nginx calls >> close() after writing the last response. However, at that time, >> that response is not fully transmitted to the client and there >> seems to be more requests not processed in the kernel buffer. >> Thus close() triggers an immediate RST. > > Thanks for the details. This looks more like the first case, and > probably can be addressed by improving likelihood of detecting the > read event. > > Could you please test if the patch below fixes the particular > issue you are seeing? It is somewhat unrelated, but it might be > a good enough solution (and is more or less equivalent to > checking r->pipeline). > >>> It would be interesting to see more details, such as tcpdump >>> and/or nginx debug logs, to find out what actually goes on here. >> >> The tcpdump and debug logs are too large to send in this mail list. >> I wonder if I can directly email it to you. > > Feel free to, my email should accept up to 100M messages. > Alternatively, a good solution might be to make the files > available for download and post a link here. > >>> Overall, given how apt uses pipelining, I tend to think that at >>> least (2) is unavoidable and can happen with certain sizes of the >>> responses. >>> >>> A good enough solution might be check for r->pipeline, which is >>> set by nginx as long as it reads a pipelined request. It might >>> not be enough though, since r->pipeline is only set for requests >>> seen by nginx as pipelined, which might not be true for the last >>> request. >>> >>> A more complete solution might be to introduce something like >>> c->pipeline flag and use lingering close if any pipelined requests >>> were seen on the connection. > > The following patch reworks handling of pipelined requests by > postponing them to the next event loop iteration. It is expected > make it more likely for nginx to know there are any additional > unread data in the socket buffer (and right now is mostly > equivalent to checking r->pipeline, since c->read->ready is always > set for pipelined requests): > > # HG changeset patch > # User Maxim Dounin > # Date 1674610218 -10800 > # Wed Jan 25 04:30:18 2023 +0300 > # Node ID 8cfd22c325a3db370b9e45aa6f897ff7bc8222f3 > # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db > Reworked pipelined requests to use posted next events. > > This is expected to improve handling of pipelined requests in a number > of ways, including: > > 1) It will make a room for additional requests from other clients, > reducing worker monopolization by a single client. > > 2) The c->read->ready flag will be set, so nginx will either read the > additional data, or will use lingering close. This is expected to help > with clients using pipelining with some constant depth, such as apt[1][2]. > > The ngx_event_move_posted_next() was modified to make it possible to > post read events on connections with kqueue. Previously, it used to > set ev->available to -1, potentially overwriting a valid positive value > provided by kqueue, so ngx_unix_recv() and ngx_readv_chain() will stop > reading from the socket before reading all the data available. > > Note that currently ngx_event_move_posted_next() will always set > the ev->ready flag. While this is expected behaviour for the ev->available > use case (where ev->ready is explicitly cleared), this is not needed for > pipelining. For pipelining, this will result in extra unneeded read() > syscall after processing of all pipelined requests, and there might be > a room for improvement here. > > [1] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 > [2] https://mailman.nginx.org/pipermail/nginx-devel/2023-January/ZA2SP5SJU55LHEBCJMFDB2AZVELRLTHI.html > > diff --git a/src/event/ngx_event_posted.c b/src/event/ngx_event_posted.c > --- a/src/event/ngx_event_posted.c > +++ b/src/event/ngx_event_posted.c > @@ -51,8 +51,10 @@ ngx_event_move_posted_next(ngx_cycle_t * > ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, > "posted next event %p", ev); > > - ev->ready = 1; > - ev->available = -1; > + if (!ev->ready) { > + ev->ready = 1; > + ev->available = -1; > + } > } > > ngx_queue_add(&ngx_posted_events, &ngx_posted_next_events); > diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c > --- a/src/http/ngx_http_request.c > +++ b/src/http/ngx_http_request.c > @@ -3129,7 +3129,7 @@ ngx_http_set_keepalive(ngx_http_request_ > } > > rev->handler = ngx_http_process_request_line; > - ngx_post_event(rev, &ngx_posted_events); > + ngx_post_event(rev, &ngx_posted_next_events); > return; > } > > > -- > Maxim Dounin > http://mdounin.ru/ > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel I can confirm that the symptom disappears after applying this patch Cheers, Miao Wang From alx.manpages at gmail.com Wed Jan 25 09:17:49 2023 From: alx.manpages at gmail.com (Alex Colomar) Date: Wed, 25 Jan 2023 10:17:49 +0100 Subject: [PATCH] QUIC: improved SO_COOKIE configure test In-Reply-To: References: <849f3b4043ee07a65bbf.1674442757@vm-bsd.mdounin.ru> <4a8c5d37-f1ff-b9ab-f47c-33a155b76480@gmail.com> Message-ID: Hello! On 1/25/23 04:19, Maxim Dounin wrote: > The inttypes.h is more portable: for example, stdint.h is not > available on FreeBSD before FreeBSD 5.0: > > https://github.com/freebsd/freebsd-src/commit/0ac2d551f20a8769869f61ebfe742fd55cef70b9 Thanks, makes sense. Cheers, Alex > > While this is not important for the particular test (since all > known Linux versions with SO_COOKIE do have stdint.h), using the > same header in all tests and the code is good from consistency > point of view. > -- -------------- next part -------------- A non-text attachment was scrubbed... Name: OpenPGP_signature Type: application/pgp-signature Size: 833 bytes Desc: OpenPGP digital signature URL: From pluknet at nginx.com Wed Jan 25 09:50:56 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Wed, 25 Jan 2023 13:50:56 +0400 Subject: [PATCH] Gzip static: ranges support (ticket #2349) In-Reply-To: References: <262ABC58-7AB2-4B53-90D4-648464DE4F44@nginx.com> Message-ID: > On 24 Jan 2023, at 06:19, Maxim Dounin wrote: > > Hello! > > On Mon, Jan 23, 2023 at 09:28:42PM +0400, Sergey Kandaurov wrote: > >>> [..] > >> On a related note, while comparing with static module, which gzip_static >> is based on, I further noticed that gzip_static doesn't check for 0-size >> response in subrequests. Existing handling of r->main suggests that >> such configuration might be used in practice, e.g. together with gunzip >> filter, as documented in the gzip_static module documentation. >> So, it makes sense to add such check for zero size buffers as well. >> >> # HG changeset patch >> # User Sergey Kandaurov >> # Date 1674493925 -14400 >> # Mon Jan 23 21:12:05 2023 +0400 >> # Node ID 27217fca1966ddb20c843384d438df2af062fdfc >> # Parent dd458c69858b88231f542be4573a3f81141d1359 >> Gzip static: avoid "zero size buf" alerts in subrequests. >> >> Similar to the static module, gzip_static enabled in subrequests might >> result in zero size buffers with responses from empty precompressed files. > > This looks like an issue introduced in 4611:2b6cb7528409, and it > might be a good idea to fix other affected modules as well. > > While valid gzipped responses are not expected to be empty, much > like valid mp4 and flv files, but it certainly shouldn't cause > "zero size buf" alerts if there is an invalid file for some > reason. > >> diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c >> --- a/src/http/modules/ngx_http_gzip_static_module.c >> +++ b/src/http/modules/ngx_http_gzip_static_module.c >> @@ -236,6 +236,10 @@ ngx_http_gzip_static_handler(ngx_http_re >> return NGX_HTTP_INTERNAL_SERVER_ERROR; >> } >> >> + if (r != r->main && of.size == 0) { >> + return ngx_http_send_header(r); >> + } >> + >> h = ngx_list_push(&r->headers_out.headers); >> if (h == NULL) { >> return NGX_HTTP_INTERNAL_SERVER_ERROR; > > While there seems to be no practical difference, I don't think > that not adding the "Content-Encoding" header for empty subrequest > responses is a good idea. I don't see either. Since it's served by the gzip_static handler, I tend to agree, in a common sense, to always attach this header. > > Further, I don't actually think that skipping sending the body, as > the static module currently does, is a good idea either. It might > break various filter modules which might not expect such > behaviour. > > I would rather consider sending the body as usual, but with > b->sync set when sending an empty body in subrequests, similarly > to how ngx_http_send_special() does. Something like this will do > the trick: > > diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c > --- a/src/http/modules/ngx_http_gzip_static_module.c > +++ b/src/http/modules/ngx_http_gzip_static_module.c > @@ -273,6 +273,7 @@ ngx_http_gzip_static_handler(ngx_http_re > b->in_file = b->file_last ? 1 : 0; > b->last_buf = (r == r->main) ? 1 : 0; > b->last_in_chain = 1; > + b->sync = (r == r->main || b->file_last) ? 0 : 1; > > b->file->fd = of.fd; > b->file->name = path; > > In particular, with this patch inflate() is able to properly > report errors on empty files: > > 2023/01/24 03:54:44 [error] 22886#100160: *1 inflate() returned -5 on response end while sending response to client, client: 127.0.0.1, server: , request: "GET /index.html HTTP/1.1", subrequest: "/empty.html", host: "127.0.0.1:8080" Note that if error is returned by any filter in subrequest, such as with gunzip, this terminates the whole request, due to c->error set. Well, this behaviour looks good enough, compared to half-baked responses. > > I also observe a similar behaviour change with empty xml files > returned in subrequests by static module and processed by the xslt > filter: previously, empty xml files were effectively ignored, and > now they result in the errors much like any other malformed xml > files. Well, that confirms that other such places are susceptible. > > Full patch: > > # HG changeset patch > # User Maxim Dounin > # Date 1674526244 -10800 > # Tue Jan 24 05:10:44 2023 +0300 > # Node ID 22e41eed77ad95f51d7d0a3daa5cb369c9643697 > # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db > Fixed "zero size buf" alerts with subrequests. > > Since 4611:2b6cb7528409 responses from the gzip static, flv, and mp4 modules > can be used with subrequests, though empty files were not properly handled. > Empty gzipped, flv, and mp4 files thus resulted in "zero size buf in output" > alerts. While valid corresponding files are not expected to be empty, such > files shouldn't result in alerts. > > Fix is to set b->sync on such empty subrequest responses, similarly to what > ngx_http_send_special() does. > > Additionally, the static module is modified to do the same instead of not > sending the response body at all in such cases, since not sending the response > body at all is believed to be at least questionable, and might break various > filters which do not expect such behaviour. > > diff --git a/src/http/modules/ngx_http_flv_module.c b/src/http/modules/ngx_http_flv_module.c > --- a/src/http/modules/ngx_http_flv_module.c > +++ b/src/http/modules/ngx_http_flv_module.c > @@ -235,6 +235,7 @@ ngx_http_flv_handler(ngx_http_request_t > b->in_file = b->file_last ? 1: 0; btw, there are old style issues originated in the static module (it was fixed in gzip_static when it was copied from there) > b->last_buf = (r == r->main) ? 1 : 0; > b->last_in_chain = 1; > + b->sync = (r == r->main || b->file_last) ? 0 : 1; > > b->file->fd = of.fd; > b->file->name = path; > diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c > --- a/src/http/modules/ngx_http_gzip_static_module.c > +++ b/src/http/modules/ngx_http_gzip_static_module.c > @@ -273,6 +273,7 @@ ngx_http_gzip_static_handler(ngx_http_re > b->in_file = b->file_last ? 1 : 0; > b->last_buf = (r == r->main) ? 1 : 0; > b->last_in_chain = 1; > + b->sync = (r == r->main || b->file_last) ? 0 : 1; > > b->file->fd = of.fd; > b->file->name = path; > diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c > --- a/src/http/modules/ngx_http_mp4_module.c > +++ b/src/http/modules/ngx_http_mp4_module.c > @@ -714,6 +714,7 @@ ngx_http_mp4_handler(ngx_http_request_t > b->in_file = b->file_last ? 1 : 0; > b->last_buf = (r == r->main) ? 1 : 0; > b->last_in_chain = 1; > + b->sync = (r == r->main || b->file_last) ? 0 : 1; > > b->file->fd = of.fd; > b->file->name = path; > diff --git a/src/http/modules/ngx_http_static_module.c b/src/http/modules/ngx_http_static_module.c > --- a/src/http/modules/ngx_http_static_module.c > +++ b/src/http/modules/ngx_http_static_module.c > @@ -238,10 +238,6 @@ ngx_http_static_handler(ngx_http_request > return NGX_HTTP_INTERNAL_SERVER_ERROR; > } > > - if (r != r->main && of.size == 0) { > - return ngx_http_send_header(r); > - } > - > r->allow_ranges = 1; > > /* we need to allocate all before the header would be sent */ > @@ -268,6 +264,7 @@ ngx_http_static_handler(ngx_http_request > b->in_file = b->file_last ? 1: 0; > b->last_buf = (r == r->main) ? 1: 0; > b->last_in_chain = 1; > + b->sync = (r == r->main || b->file_last) ? 0 : 1; > > b->file->fd = of.fd; > b->file->name = path; > In addition to the static module change, any reason not to include ngx_http_cache_send()? It has a similar behaviour to skip cached empty body if served in subrequests. Yet another similar place is subrequests processed with rewrite such as "return 204;". diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -1803,10 +1803,6 @@ ngx_http_send_response(ngx_http_request_ } } - if (r != r->main && val.len == 0) { - return ngx_http_send_header(r); - } - b = ngx_calloc_buf(r->pool); if (b == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; @@ -1817,6 +1813,7 @@ ngx_http_send_response(ngx_http_request_ b->memory = val.len ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (r == r->main || val.len) ? 0 : 1; out.buf = b; out.next = NULL; diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -1575,10 +1575,6 @@ ngx_http_cache_send(ngx_http_request_t * ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http file cache send: %s", c->file.name.data); - if (r != r->main && c->length - c->body_start == 0) { - return ngx_http_send_header(r); - } - /* we need to allocate all before the header would be sent */ b = ngx_calloc_buf(r->pool); @@ -1603,6 +1599,7 @@ ngx_http_cache_send(ngx_http_request_t * b->in_file = (c->length - c->body_start) ? 1: 0; b->last_buf = (r == r->main) ? 1: 0; b->last_in_chain = 1; + b->sync = (r == r->main || b->file_last - b->file_pos) ? 0 : 1; b->file->fd = c->file.fd; b->file->name = c->file.name; -- Sergey Kandaurov From pluknet at nginx.com Wed Jan 25 12:08:07 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Wed, 25 Jan 2023 16:08:07 +0400 Subject: [PATCH] Fixed handling of very long locations (ticket #2435) In-Reply-To: <4af6cc78dc72fbb15326.1674071423@vm-bsd.mdounin.ru> References: <4af6cc78dc72fbb15326.1674071423@vm-bsd.mdounin.ru> Message-ID: <050D0524-C524-42E4-AEF2-C895B921CBB5@nginx.com> > On 18 Jan 2023, at 23:50, Maxim Dounin wrote: > > # HG changeset patch > # User Maxim Dounin > # Date 1674071317 -10800 > # Wed Jan 18 22:48:37 2023 +0300 > # Node ID 4af6cc78dc72fbb15326e6ffbbd30935f0cb794b > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > Fixed handling of very long locations (ticket #2435). > > Previously, location prefix length in ngx_http_location_tree_node_t was > stored as "u_char", and therefore location prefixes longer than 255 bytes > were handled incorrectly. > > Fix is to use "u_short" instead. With "u_short", prefixes up to 65535 bytes > can be safely used, and this isn't reachable due to NGX_CONF_BUFFER, which > is 4096 bytes. > > diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c > --- a/src/http/ngx_http.c > +++ b/src/http/ngx_http.c > @@ -1130,7 +1130,7 @@ ngx_http_create_locations_tree(ngx_conf_ > node->auto_redirect = (u_char) ((lq->exact && lq->exact->auto_redirect) > || (lq->inclusive && lq->inclusive->auto_redirect)); > > - node->len = (u_char) len; > + node->len = (u_short) len; > ngx_memcpy(node->name, &lq->name->data[prefix], len); > > ngx_queue_split(locations, q, &tail); > diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h > --- a/src/http/ngx_http_core_module.h > +++ b/src/http/ngx_http_core_module.h > @@ -463,8 +463,8 @@ struct ngx_http_location_tree_node_s { > ngx_http_core_loc_conf_t *exact; > ngx_http_core_loc_conf_t *inclusive; > > + u_short len; > u_char auto_redirect; > - u_char len; > u_char name[1]; > }; > Looks good. -- Sergey Kandaurov From mdounin at mdounin.ru Wed Jan 25 21:29:28 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 26 Jan 2023 00:29:28 +0300 Subject: [PATCH] Added warning about redefinition of listen socket protocol options In-Reply-To: <5450B60D-DD89-4861-BA4D-6B57F085F7FD@nginx.com> References: <5450B60D-DD89-4861-BA4D-6B57F085F7FD@nginx.com> Message-ID: Hello! On Mon, Jan 23, 2023 at 02:21:33PM +0400, Sergey Kandaurov wrote: > > On 31 Dec 2022, at 18:35, Maxim Dounin wrote: > > > > # HG changeset patch > > # User Maxim Dounin > > # Date 1672497248 -10800 > > # Sat Dec 31 17:34:08 2022 +0300 > > # Node ID c215d5cf25732ece1819cf1cd48ebb480bb642c7 > > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > > Added warning about redefinition of listen socket protocol options. > > > > The "listen" directive in the http module can be used multiple times > > in different server blocks. Originally, it was supposed to be specified > > once with various socket options, and without any parameters in virtual > > server blocks. For example: > > > > server { listen 80 backlog=1024; server_name foo; ... } > > server { listen 80; server_name bar; ... } > > server { listen 80; server_name bazz; ... } > > > > The address part of the syntax ("address[:port]" / "port" / "unix:path") > > uniquely identifies the listening socket, and therefore is enough for > > name-based virtual servers (to let nginx know that the virtual server > > accepts requests on the listening socket in question). > > > > To ensure that listening options do not conflict between virtual servers, > > they were allowed only once. For example, the following configuration > > will be rejected ("duplicate listen options for 0.0.0.0:80 in ..."): > > > > server { listen 80 backlog=1024; server_name foo; ... } > > server { listen 80 backlog=512; server_name bar; ... } > > > > At some point it was, however, noticed, that it is sometimes convenient > > to repeat some options for clarity. In nginx 0.8.51 the "ssl" parameter > > was allowed to be specified multiple times, e.g.: > > > > server { listen 443 ssl backlog=1024; server_name foo; ... } > > server { listen 443 ssl; server_name bar; ... } > > server { listen 443 ssl; server_name bazz; ... } > > > > This approach makes configuration more readable, since SSL sockets are > > immediately visible in the configuration. If this is not needed, just the > > address can still be used. > > > > Later, additional protocol-specific options similar to "ssl" were > > introduced, notably "http2" and "proxy_protocol". With these options, > > one can write: > > > > server { listen 443 ssl backlog=1024; server_name foo; ... } > > server { listen 443 http2; server_name bar; ... } > > server { listen 443 proxy_protocol; server_name bazz; ... } > > > > The resulting socket will use ssl, http2, and proxy_protocol, but this > > is not really obvious from the configuration. > > > > To ensure such misleading configurations are not allowed, nginx now > > warns as long as the "listen" directive is used with options different > > nitpicking: > > "ensure .. are not allowed" and "warns" don't seem to be equally strong. > As such, I'd rewrite to something like: > > To emphasize such misleading configurations are discouraged, nginx now Thanks, changed. > > from the options previously used if these are potentially confusing. > > Not really confident what "these" refers to. > > s/these/they/ ? I don't think that there is much difference between "these" and "they" when it comes to what they refer to. Either way, "if this is potentially confusing" is probably better and unambiguously refers to the fact that the "listen" directive is used with the different options. Changed. > > In particular, the following configurations are allowed: > > > > server { listen 8401 ssl backlog=1024; server_name foo; } > > server { listen 8401 ssl; server_name bar; } > > server { listen 8401 ssl; server_name bazz; } > > > > server { listen 8402 ssl http2 backlog=1024; server_name foo; } > > server { listen 8402 ssl; server_name bar; } > > server { listen 8402 ssl; server_name bazz; } > > > > server { listen 8403 ssl; server_name bar; } > > server { listen 8403 ssl; server_name bazz; } > > server { listen 8403 ssl http2; server_name foo; } > > > > server { listen 8404 ssl http2 backlog=1024; server_name foo; } > > server { listen 8404 http2; server_name bar; } > > server { listen 8404 http2; server_name bazz; } > > > > server { listen 8405 ssl http2 backlog=1024; server_name foo; } > > server { listen 8405 ssl http2; server_name bar; } > > server { listen 8405 ssl http2; server_name bazz; } > > > > server { listen 8406 ssl; server_name foo; } > > server { listen 8406; server_name bar; } > > server { listen 8406; server_name bazz; } > > > > And the following configurations will generate warnings: > > > > server { listen 8501 ssl http2 backlog=1024; server_name foo; } > > server { listen 8501 http2; server_name bar; } > > server { listen 8501 ssl; server_name bazz; } > > > > server { listen 8502 backlog=1024; server_name foo; } > > server { listen 8502 ssl; server_name bar; } > > > > server { listen 8503 ssl; server_name foo; } > > server { listen 8503 http2; server_name bar; } > > > > server { listen 8504 ssl; server_name foo; } > > server { listen 8504 http2; server_name bar; } > > server { listen 8504 proxy_protocol; server_name bazz; } > > > > server { listen 8505 ssl http2 proxy_protocol; server_name foo; } > > server { listen 8505 ssl http2; server_name bar; } > > server { listen 8505 ssl; server_name bazz; } > > > > server { listen 8506 ssl http2; server_name foo; } > > server { listen 8506 ssl; server_name bar; } > > server { listen 8506; server_name bazz; } > > > > server { listen 8507 ssl; server_name bar; } > > server { listen 8507; server_name bazz; } > > server { listen 8507 ssl http2; server_name foo; } > > > > server { listen 8508 ssl; server_name bar; } > > server { listen 8508; server_name bazz; } > > server { listen 8508 ssl backlog=1024; server_name foo; } > > > > server { listen 8509; server_name bazz; } > > server { listen 8509 ssl; server_name bar; } > > server { listen 8509 ssl backlog=1024; server_name foo; } > > > > 15 examples of dos and don'ts looks slightly excessive. > The accurate description (such as provided by you below) allows > to reduce most of them to e.g. four common invalid configurations: > > A lesser option set with socket option: > > server { listen 8443 backlog=1024; server_name foo; } > server { listen 8443 http2; server_name bar; } > > The main option set is repeated at least twice: > > server { listen 127.0.0.1:8443; server_name foo; } > server { listen 127.0.0.1:8443 ssl; server_name bar; } > server { listen 127.0.0.1:8443 ssl; server_name baz; } > > Option sets partially overlap: > > server { listen 127.0.0.1:8443 ssl; server_name foo; } > server { listen 127.0.0.1:8443 http2; server_name bar; } > > More than two option sets: > > server { listen 127.0.0.1:8443 http2 ssl; server_name foo; } > server { listen 127.0.0.1:8443 http2; server_name bar; } > server { listen 127.0.0.1:8443 ssl; server_name baz; } While your approach might be better from documentation point of view, the way it is currently described in the commit log is how it was designed: from the examples of valid and invalid configurations. My current working tests contain 18 valid and 22 invalid configurations, derived from the ones provided in the commit log with additional shuffling. But since these are derived I've decided to avoid providing all of them in the commit log. > > The basic idea is that at most two sets of protocol options are allowed: > > the main one (with socket options, if any), and a shorter one, with options > > being a subset of the main options, repeated for clarity. As long as the > > shorter set of protocol options is used, all listen directives except the > > main one should use it. > > I'd move this paragraph somewhere before examples, as this is the most > specific description of things actually changed. This paragraph summarizes changes made to address the issue described, and I don't think moving it will improve things. > BTW, while reviewing I caught sort of a bug. > As I understand the above explanation, if there are both full and short > sets present, then at most one listen directive can have the full set, > while shorter sets can be repeated. If so, then with the proposed patch > the next configuration is expectedly invalid: > > server { listen 127.0.0.1:8443 ssl; server_name foo; } > server { listen 127.0.0.1:8443 ssl; server_name bar; } > server { listen 127.0.0.1:8443; server_name baz; } > > This is expected since first two servers with same options are > interpreted as a short form (with full form seen potentially later on), > but 3rd server has lesser options, which is caught by this check: > (addr[i].protocols_set && protocols != addr[i].protocols) > Which is interpreted as: > "this server has a lesser set that doesn't match a a shorter set". > > Now, if 3rd server is moved first, configuration starts to pass: > > server { listen 127.0.0.1:8443; server_name baz; } > server { listen 127.0.0.1:8443 ssl; server_name foo; } > server { listen 127.0.0.1:8443 ssl; server_name bar; } > > This is because after (now) 2nd server, it is parsed as: > 1st server has a short form, and 2nd server has a full form. > Then 3rd server goes to "the same options" case. This also > overwrites the remembered shorter set in addr[i].protocols. > > I guess an additional check should be added to address this. > It is similar to the "options removed" case and ensures that > the repeated options set actually matches a shorter set: > > diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c > --- a/src/http/ngx_http.c > +++ b/src/http/ngx_http.c > @@ -1345,7 +1345,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, n > > /* the same options */ > > - if (lsopt->set && addr[i].protocols_changed) { > + if ((lsopt->set && addr[i].protocols_changed) > + || (addr[i].protocols_set && protocols != addr[i].protocols)) > + { > ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > "protocol options redefined for %V", > &addr[i].opt.addr_text); Thanks for catching this, applied. Updated patch: # HG changeset patch # User Maxim Dounin # Date 1674624978 -10800 # Wed Jan 25 08:36:18 2023 +0300 # Node ID 1619d1563c3231d4283937610ce97aa1e3824fb8 # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db Added warning about redefinition of listen socket protocol options. The "listen" directive in the http module can be used multiple times in different server blocks. Originally, it was supposed to be specified once with various socket options, and without any parameters in virtual server blocks. For example: server { listen 80 backlog=1024; server_name foo; ... } server { listen 80; server_name bar; ... } server { listen 80; server_name bazz; ... } The address part of the syntax ("address[:port]" / "port" / "unix:path") uniquely identifies the listening socket, and therefore is enough for name-based virtual servers (to let nginx know that the virtual server accepts requests on the listening socket in question). To ensure that listening options do not conflict between virtual servers, they were allowed only once. For example, the following configuration will be rejected ("duplicate listen options for 0.0.0.0:80 in ..."): server { listen 80 backlog=1024; server_name foo; ... } server { listen 80 backlog=512; server_name bar; ... } At some point it was, however, noticed, that it is sometimes convenient to repeat some options for clarity. In nginx 0.8.51 the "ssl" parameter was allowed to be specified multiple times, e.g.: server { listen 443 ssl backlog=1024; server_name foo; ... } server { listen 443 ssl; server_name bar; ... } server { listen 443 ssl; server_name bazz; ... } This approach makes configuration more readable, since SSL sockets are immediately visible in the configuration. If this is not needed, just the address can still be used. Later, additional protocol-specific options similar to "ssl" were introduced, notably "http2" and "proxy_protocol". With these options, one can write: server { listen 443 ssl backlog=1024; server_name foo; ... } server { listen 443 http2; server_name bar; ... } server { listen 443 proxy_protocol; server_name bazz; ... } The resulting socket will use ssl, http2, and proxy_protocol, but this is not really obvious from the configuration. To emphasize such misleading configurations are discouraged, nginx now warns as long as the "listen" directive is used with options different from the options previously used if this is potentially confusing. In particular, the following configurations are allowed: server { listen 8401 ssl backlog=1024; server_name foo; } server { listen 8401 ssl; server_name bar; } server { listen 8401 ssl; server_name bazz; } server { listen 8402 ssl http2 backlog=1024; server_name foo; } server { listen 8402 ssl; server_name bar; } server { listen 8402 ssl; server_name bazz; } server { listen 8403 ssl; server_name bar; } server { listen 8403 ssl; server_name bazz; } server { listen 8403 ssl http2; server_name foo; } server { listen 8404 ssl http2 backlog=1024; server_name foo; } server { listen 8404 http2; server_name bar; } server { listen 8404 http2; server_name bazz; } server { listen 8405 ssl http2 backlog=1024; server_name foo; } server { listen 8405 ssl http2; server_name bar; } server { listen 8405 ssl http2; server_name bazz; } server { listen 8406 ssl; server_name foo; } server { listen 8406; server_name bar; } server { listen 8406; server_name bazz; } And the following configurations will generate warnings: server { listen 8501 ssl http2 backlog=1024; server_name foo; } server { listen 8501 http2; server_name bar; } server { listen 8501 ssl; server_name bazz; } server { listen 8502 backlog=1024; server_name foo; } server { listen 8502 ssl; server_name bar; } server { listen 8503 ssl; server_name foo; } server { listen 8503 http2; server_name bar; } server { listen 8504 ssl; server_name foo; } server { listen 8504 http2; server_name bar; } server { listen 8504 proxy_protocol; server_name bazz; } server { listen 8505 ssl http2 proxy_protocol; server_name foo; } server { listen 8505 ssl http2; server_name bar; } server { listen 8505 ssl; server_name bazz; } server { listen 8506 ssl http2; server_name foo; } server { listen 8506 ssl; server_name bar; } server { listen 8506; server_name bazz; } server { listen 8507 ssl; server_name bar; } server { listen 8507; server_name bazz; } server { listen 8507 ssl http2; server_name foo; } server { listen 8508 ssl; server_name bar; } server { listen 8508; server_name bazz; } server { listen 8508 ssl backlog=1024; server_name foo; } server { listen 8509; server_name bazz; } server { listen 8509 ssl; server_name bar; } server { listen 8509 ssl backlog=1024; server_name foo; } The basic idea is that at most two sets of protocol options are allowed: the main one (with socket options, if any), and a shorter one, with options being a subset of the main options, repeated for clarity. As long as the shorter set of protocol options is used, all listen directives except the main one should use it. diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c --- a/src/http/ngx_http.c +++ b/src/http/ngx_http.c @@ -1228,7 +1228,8 @@ static ngx_int_t ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, ngx_http_conf_port_t *port, ngx_http_listen_opt_t *lsopt) { - ngx_uint_t i, default_server, proxy_protocol; + ngx_uint_t i, default_server, proxy_protocol, + protocols, protocols_prev; ngx_http_conf_addr_t *addr; #if (NGX_HTTP_SSL) ngx_uint_t ssl; @@ -1264,12 +1265,18 @@ ngx_http_add_addresses(ngx_conf_t *cf, n default_server = addr[i].opt.default_server; proxy_protocol = lsopt->proxy_protocol || addr[i].opt.proxy_protocol; + protocols = lsopt->proxy_protocol; + protocols_prev = addr[i].opt.proxy_protocol; #if (NGX_HTTP_SSL) ssl = lsopt->ssl || addr[i].opt.ssl; + protocols |= lsopt->ssl << 1; + protocols_prev |= addr[i].opt.ssl << 1; #endif #if (NGX_HTTP_V2) http2 = lsopt->http2 || addr[i].opt.http2; + protocols |= lsopt->http2 << 2; + protocols_prev |= addr[i].opt.http2 << 2; #endif if (lsopt->set) { @@ -1299,6 +1306,57 @@ ngx_http_add_addresses(ngx_conf_t *cf, n addr[i].default_server = cscf; } + /* check for conflicting protocol options */ + + if ((protocols | protocols_prev) != protocols_prev) { + + /* options added */ + + if ((addr[i].opt.set && !lsopt->set) + || addr[i].protocols_changed + || (protocols | protocols_prev) != protocols) + { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "protocol options redefined for %V", + &addr[i].opt.addr_text); + } + + addr[i].protocols = protocols_prev; + addr[i].protocols_set = 1; + addr[i].protocols_changed = 1; + + } else if ((protocols_prev | protocols) != protocols) { + + /* options removed */ + + if (lsopt->set + || (addr[i].protocols_set && protocols != addr[i].protocols)) + { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "protocol options redefined for %V", + &addr[i].opt.addr_text); + } + + addr[i].protocols = protocols; + addr[i].protocols_set = 1; + addr[i].protocols_changed = 1; + + } else { + + /* the same options */ + + if ((lsopt->set && addr[i].protocols_changed) + || (addr[i].protocols_set && protocols != addr[i].protocols)) + { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "protocol options redefined for %V", + &addr[i].opt.addr_text); + } + + addr[i].protocols = protocols; + addr[i].protocols_set = 1; + } + addr[i].opt.default_server = default_server; addr[i].opt.proxy_protocol = proxy_protocol; #if (NGX_HTTP_SSL) @@ -1355,6 +1413,9 @@ ngx_http_add_address(ngx_conf_t *cf, ngx } addr->opt = *lsopt; + addr->protocols = 0; + addr->protocols_set = 0; + addr->protocols_changed = 0; addr->hash.buckets = NULL; addr->hash.size = 0; addr->wc_head = NULL; diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -274,6 +274,10 @@ typedef struct { typedef struct { ngx_http_listen_opt_t opt; + unsigned protocols:3; + unsigned protocols_set:1; + unsigned protocols_changed:1; + ngx_hash_t hash; ngx_hash_wildcard_t *wc_head; ngx_hash_wildcard_t *wc_tail; -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Thu Jan 26 02:30:19 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 26 Jan 2023 05:30:19 +0300 Subject: [PATCH] Fixed handling of very long locations (ticket #2435) In-Reply-To: <050D0524-C524-42E4-AEF2-C895B921CBB5@nginx.com> References: <4af6cc78dc72fbb15326.1674071423@vm-bsd.mdounin.ru> <050D0524-C524-42E4-AEF2-C895B921CBB5@nginx.com> Message-ID: Hello! On Wed, Jan 25, 2023 at 04:08:07PM +0400, Sergey Kandaurov wrote: > > On 18 Jan 2023, at 23:50, Maxim Dounin wrote: > > > > # HG changeset patch > > # User Maxim Dounin > > # Date 1674071317 -10800 > > # Wed Jan 18 22:48:37 2023 +0300 > > # Node ID 4af6cc78dc72fbb15326e6ffbbd30935f0cb794b > > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > > Fixed handling of very long locations (ticket #2435). > > > > Previously, location prefix length in ngx_http_location_tree_node_t was > > stored as "u_char", and therefore location prefixes longer than 255 bytes > > were handled incorrectly. > > > > Fix is to use "u_short" instead. With "u_short", prefixes up to 65535 bytes > > can be safely used, and this isn't reachable due to NGX_CONF_BUFFER, which > > is 4096 bytes. > > > > diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c > > --- a/src/http/ngx_http.c > > +++ b/src/http/ngx_http.c > > @@ -1130,7 +1130,7 @@ ngx_http_create_locations_tree(ngx_conf_ > > node->auto_redirect = (u_char) ((lq->exact && lq->exact->auto_redirect) > > || (lq->inclusive && lq->inclusive->auto_redirect)); > > > > - node->len = (u_char) len; > > + node->len = (u_short) len; > > ngx_memcpy(node->name, &lq->name->data[prefix], len); > > > > ngx_queue_split(locations, q, &tail); > > diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h > > --- a/src/http/ngx_http_core_module.h > > +++ b/src/http/ngx_http_core_module.h > > @@ -463,8 +463,8 @@ struct ngx_http_location_tree_node_s { > > ngx_http_core_loc_conf_t *exact; > > ngx_http_core_loc_conf_t *inclusive; > > > > + u_short len; > > u_char auto_redirect; > > - u_char len; > > u_char name[1]; > > }; > > > > Looks good. Thanks for the review, pushed to http://mdounin.ru/hg/nginx. -- Maxim Dounin http://mdounin.ru/ From xeioex at nginx.com Thu Jan 26 05:59:58 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 26 Jan 2023 05:59:58 +0000 Subject: [njs] Added njs_vm_value_string_create() and njs_vm_value_string_create_chb(). Message-ID: details: https://hg.nginx.org/njs/rev/742347841e34 branches: changeset: 2027:742347841e34 user: Dmitry Volyntsev date: Fri Jan 20 20:15:03 2023 -0800 description: Added njs_vm_value_string_create() and njs_vm_value_string_create_chb(). Unlike njs_vm_value_string_set() the new functions always check the input string for a valid UTF8 and calculate UTF8 character length by themselves. diffstat: src/njs.h | 4 ++++ src/njs_chb.h | 2 +- src/njs_json.c | 11 +++-------- src/njs_regexp.c | 18 +++--------------- src/njs_string.c | 46 ++++++++++++++++++++++++++++++++-------------- src/njs_string.h | 2 ++ src/njs_vm.c | 16 ++++++++++++++++ src/test/njs_unit_test.c | 15 --------------- 8 files changed, 61 insertions(+), 53 deletions(-) diffs (255 lines): diff -r 8b9414a6e557 -r 742347841e34 src/njs.h --- a/src/njs.h Wed Jan 18 18:33:23 2023 -0800 +++ b/src/njs.h Fri Jan 20 20:15:03 2023 -0800 @@ -411,6 +411,10 @@ NJS_EXPORT njs_int_t njs_vm_value_string const u_char *start, uint32_t size); NJS_EXPORT u_char *njs_vm_value_string_alloc(njs_vm_t *vm, njs_value_t *value, uint32_t size); +NJS_EXPORT njs_int_t njs_vm_value_string_create(njs_vm_t *vm, + njs_value_t *value, const u_char *start, uint32_t size); +NJS_EXPORT njs_int_t njs_vm_value_string_create_chb(njs_vm_t *vm, + njs_value_t *value, njs_chb_t *chain); NJS_EXPORT njs_int_t njs_vm_value_string_copy(njs_vm_t *vm, njs_str_t *retval, njs_value_t *value, uintptr_t *next); NJS_EXPORT njs_int_t njs_vm_string_compare(const njs_value_t *v1, diff -r 8b9414a6e557 -r 742347841e34 src/njs_chb.h --- a/src/njs_chb.h Wed Jan 18 18:33:23 2023 -0800 +++ b/src/njs_chb.h Fri Jan 20 20:15:03 2023 -0800 @@ -105,7 +105,7 @@ njs_chb_utf8_length(njs_chb_t *chain) while (n != NULL) { len = njs_utf8_length(n->start, njs_chb_node_size(n)); if (njs_slow_path(len < 0)) { - return 0; + return -1; } length += len; diff -r 8b9414a6e557 -r 742347841e34 src/njs_json.c --- a/src/njs_json.c Wed Jan 18 18:33:23 2023 -0800 +++ b/src/njs_json.c Fri Jan 20 20:15:03 2023 -0800 @@ -1090,8 +1090,7 @@ static njs_int_t njs_json_stringify_iterator(njs_vm_t *vm, njs_json_stringify_t *stringify, njs_value_t *object) { - u_char *p; - int64_t size, length; + int64_t size; njs_int_t ret; njs_chb_t chain; njs_value_t *key, *value, index, wrapper; @@ -1224,16 +1223,12 @@ done: goto release; } - length = njs_chb_utf8_length(&chain); - - p = njs_string_alloc(vm, &vm->retval, size, length); - if (njs_slow_path(p == NULL)) { + ret = njs_string_create_chb(vm, &vm->retval, &chain); + if (njs_slow_path(ret != NJS_OK)) { njs_chb_destroy(&chain); goto memory_error; } - njs_chb_join_to(&chain, p); - release: njs_chb_destroy(&chain); diff -r 8b9414a6e557 -r 742347841e34 src/njs_regexp.c --- a/src/njs_regexp.c Wed Jan 18 18:33:23 2023 -0800 +++ b/src/njs_regexp.c Fri Jan 20 20:15:03 2023 -0800 @@ -1197,8 +1197,7 @@ static njs_int_t njs_regexp_prototype_symbol_replace(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused) { - u_char *p; - int64_t n, last_index, ncaptures, pos, next_pos, size, length; + int64_t n, last_index, ncaptures, pos, next_pos, length; njs_str_t rep, m; njs_int_t ret; njs_arr_t results; @@ -1455,23 +1454,12 @@ njs_regexp_prototype_symbol_replace(njs_ njs_chb_append(&chain, &s.start[next_pos], s.size - next_pos); } - size = njs_chb_size(&chain); - if (njs_slow_path(size < 0)) { - njs_memory_error(vm); + ret = njs_string_create_chb(vm, &vm->retval, &chain); + if (njs_slow_path(ret != NJS_OK)) { ret = NJS_ERROR; goto exception; } - length = njs_chb_utf8_length(&chain); - - p = njs_string_alloc(vm, &vm->retval, size, length); - if (njs_slow_path(p == NULL)) { - ret = NJS_ERROR; - goto exception; - } - - njs_chb_join_to(&chain, p); - ret = NJS_OK; exception: diff -r 8b9414a6e557 -r 742347841e34 src/njs_string.c --- a/src/njs_string.c Wed Jan 18 18:33:23 2023 -0800 +++ b/src/njs_string.c Fri Jan 20 20:15:03 2023 -0800 @@ -147,6 +147,35 @@ njs_string_create(njs_vm_t *vm, njs_valu njs_int_t +njs_string_create_chb(njs_vm_t *vm, njs_value_t *value, njs_chb_t *chain) +{ + u_char *p; + ssize_t size, length; + + size = njs_chb_size(chain); + if (njs_slow_path(size < 0)) { + njs_memory_error(vm); + return NJS_ERROR; + } + + length = njs_chb_utf8_length(chain); + if (njs_slow_path(length < 0)) { + njs_memory_error(vm); + return NJS_ERROR; + } + + p = njs_string_alloc(vm, value, size, length); + if (njs_slow_path(p == NULL)) { + return NJS_ERROR; + } + + njs_chb_join_to(chain, p); + + return NJS_OK; +} + + +njs_int_t njs_string_new(njs_vm_t *vm, njs_value_t *value, const u_char *start, uint32_t size, uint32_t length) { @@ -3489,7 +3518,7 @@ njs_string_get_substitution(njs_vm_t *vm njs_value_t *string, int64_t pos, njs_value_t *captures, int64_t ncaptures, njs_value_t *groups, njs_value_t *replacement, njs_value_t *retval) { - int64_t tail, size, length, n; + int64_t tail, n; u_char c, c2, *p, *r, *end; njs_str_t rep, m, str, cap; njs_int_t ret; @@ -3620,23 +3649,12 @@ njs_string_get_substitution(njs_vm_t *vm done: - size = njs_chb_size(&chain); - if (njs_slow_path(size < 0)) { - njs_memory_error(vm); + ret = njs_string_create_chb(vm, retval, &chain); + if (njs_slow_path(ret != NJS_OK)) { ret = NJS_ERROR; goto exception; } - length = njs_chb_utf8_length(&chain); - - p = njs_string_alloc(vm, retval, size, length); - if (njs_slow_path(p == NULL)) { - ret = NJS_ERROR; - goto exception; - } - - njs_chb_join_to(&chain, p); - ret = NJS_OK; exception: diff -r 8b9414a6e557 -r 742347841e34 src/njs_string.h --- a/src/njs_string.h Wed Jan 18 18:33:23 2023 -0800 +++ b/src/njs_string.h Fri Jan 20 20:15:03 2023 -0800 @@ -200,6 +200,8 @@ njs_int_t njs_string_new(njs_vm_t *vm, n uint32_t size, uint32_t length); njs_int_t njs_string_create(njs_vm_t *vm, njs_value_t *value, const char *src, size_t size); +njs_int_t njs_string_create_chb(njs_vm_t *vm, njs_value_t *value, + njs_chb_t *chain); void njs_encode_hex(njs_str_t *dst, const njs_str_t *src); size_t njs_encode_hex_length(const njs_str_t *src, size_t *out_size); diff -r 8b9414a6e557 -r 742347841e34 src/njs_vm.c --- a/src/njs_vm.c Wed Jan 18 18:33:23 2023 -0800 +++ b/src/njs_vm.c Fri Jan 20 20:15:03 2023 -0800 @@ -832,6 +832,22 @@ njs_vm_value_string_alloc(njs_vm_t *vm, } +njs_int_t +njs_vm_value_string_create(njs_vm_t *vm, njs_value_t *value, + const u_char *start, uint32_t size) +{ + return njs_string_create(vm, value, (const char *) start, size); +} + + +njs_int_t +njs_vm_value_string_create_chb(njs_vm_t *vm, njs_value_t *value, + njs_chb_t *chain) +{ + return njs_string_create_chb(vm, value, chain); +} + + njs_function_t * njs_vm_function(njs_vm_t *vm, const njs_str_t *path) { diff -r 8b9414a6e557 -r 742347841e34 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Wed Jan 18 18:33:23 2023 -0800 +++ b/src/test/njs_unit_test.c Fri Jan 20 20:15:03 2023 -0800 @@ -8984,9 +8984,6 @@ static njs_unit_test_t njs_test[] = { njs_str("'abc'.replace(/b/g, '$0')"), njs_str("a$0c") }, - { njs_str("typeof String.bytesFrom(Array(15).fill(0xE3)).replace(/^/g, 1)"), - njs_str("string") }, - { njs_str("'abc'.replace(/^/g, '|$&|')"), njs_str("||abc") }, @@ -9060,12 +9057,6 @@ static njs_unit_test_t njs_test[] = "r[Symbol.replace]('foo', function() {m = arguments[0]}); [m, typeof m]"), njs_str("undefined,string") }, - { njs_str("String.bytesFrom([253,242,141,10]).replace(/\\s/g, 'X')[3]"), - njs_str("X") }, - - { njs_str("String.bytesFrom([255,149,15,97,95]).replace(/_/g, 'X')[4]"), - njs_str("X") }, - { njs_str("var a = [];" "a[2] = '';" "var re = /any_regexp/;" @@ -9271,9 +9262,6 @@ static njs_unit_test_t njs_test[] = "a +' '+ a.length"), njs_str("αα 4") }, - { njs_str("typeof String.bytesFrom(Array(15).fill(0xE3)).match(/^/g)"), - njs_str("object") }, - { njs_str("'abc'.split()"), njs_str("abc") }, @@ -18082,9 +18070,6 @@ static njs_unit_test_t njs_test[] = { njs_str("JSON.stringify('\\u00CE\\u00B1\\u00C2\\u00B6'.toBytes())"), njs_str("\"α¶\"") }, - { njs_str("JSON.stringify('µ§±®'.toBytes())"), - njs_str("\"\xB5\xA7\xB1\xAE\"") }, - /* Optional arguments. */ { njs_str("JSON.stringify(undefined, undefined, 1)"), From xeioex at nginx.com Thu Jan 26 06:00:00 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 26 Jan 2023 06:00:00 +0000 Subject: [njs] Added "xml" module for working with XML documents. Message-ID: details: https://hg.nginx.org/njs/rev/99b9f83e4d4d branches: changeset: 2028:99b9f83e4d4d user: Dmitry Volyntsev date: Wed Jan 25 21:54:47 2023 -0800 description: Added "xml" module for working with XML documents. - xml.parse(string|buffer) returns an XMLDoc wrapper object around XML structure. - xml.c14n(root_node[, excluding_node]]) canonicalizes root_node and its children according to https://www.w3.org/TR/xml-c14n, optionally excluding_node allows to omit from the output a part of the document. - xml.exclusiveC14n(root_node[, excluding_node[, withComments [, prefix_list]]]) canonicalizes root_node and its children according to https://www.w3.org/TR/xml-exc-c14n/. excluding_node allows to omit from the output a part of the document corresponding to the node and its children. withComments is a boolean and is false by default. When withComments is true canonicalization corresponds to http://www.w3.org/2001/10/xml-exc-c14n#WithComments. prefix_list is an optional string with a space separated namespace prefixes for namespaces that should also be included into the output. - XMLDoc an XMLDoc wrapper object around XML structure. doc.xxx returns the first root tag named "xxx" as XMLNode wrapper object. - XMLNode an XMLNode wrapper object around XML tag node. node.$tag$xxx returns the first child tag named "xxx" as XMLNode wrapper object. node.xxx a shorthand syntax for node.$tag$xxx. node.$tags$xxx? returns an array of all children tags named xxx. node.$attr$xxx returns an attribute value of xxx. node.$attrs returns an XMLAttr wrapper object. node.$name returns the tag name of the node. node.$ns returns the namespace of the node. node.$parent returns the parent of the node. node.$text returns the node's content. - XMLAttrs an XMLAttrs wrapper object around XML node attributes. attrs.xxx returns a value of the xxx attribute. - Example: const xml = require("xml"); let data = `ToveJani`; let doc = xml.parse(data); console.log(doc.note.to.$text) /* 'Tove' */ console.log(doc.note.to.$attr$b) /* 'bar' */ console.log(doc.note.$tags[1].$text) /* 'Jani' */ let dec = new TextDecoder(); let c14n = dec.decode(xml.exclusiveC14n(doc.note)); console.log(c14n) /* 'ToveJani' */ c14n = dec.decode(xml.exclusiveC14n(doc.note.to)); console.log(c14n) /* 'Tove' */ c14n = dec.decode(xml.exclusiveC14n(doc.note, doc.note.to /* excluding 'to' */)); console.log(c14n) /* 'Jani' */ diffstat: auto/libxml2 | 77 + auto/modules | 8 + auto/options | 2 + configure | 1 + external/njs_xml_module.c | 1322 ++++++++++++++++++++ nginx/config | 5 +- nginx/config.make | 2 +- nginx/ngx_js.c | 2 + src/test/njs_unit_test.c | 93 + test/harness/compatNjs.js | 18 + test/harness/compatXml.js | 8 + test/xml/README.rst | 26 + test/xml/auth_r.xml | 27 + test/xml/auth_r_prefix_list.xml | 28 + test/xml/auth_r_prefix_list_signed.xml | 23 + test/xml/auth_r_signed.xml | 22 + test/xml/auth_r_signed2.xml | 22 + test/xml/auth_r_with_comments_signed.xml | 22 + test/xml/example.com.crt | 13 + test/xml/response_assertion_and_message_signed.xml | 46 + test/xml/response_signed.xml | 42 + test/xml/response_signed_broken.xml | 42 + test/xml/response_signed_broken2.xml | 42 + test/xml/saml_verify.t.js | 228 +++ 24 files changed, 2118 insertions(+), 3 deletions(-) diffs (truncated from 2287 to 1000 lines): diff -r 742347841e34 -r 99b9f83e4d4d auto/libxml2 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/auto/libxml2 Wed Jan 25 21:54:47 2023 -0800 @@ -0,0 +1,77 @@ + +# Copyright (C) Dmitry Volyntsev +# Copyright (C) NGINX, Inc. + +NJS_HAVE_LIBXML2=NO + +if [ $NJS_LIBXML2 = YES ]; then + njs_found=no + + njs_feature="libxml2" + njs_feature_name=NJS_HAVE_LIBXML2 + njs_feature_run=no + njs_feature_incs="/usr/include/libxml2" + njs_feature_libs="-lxml2" + njs_feature_test="#include + #include + + int main() { + xmlDocPtr tree; + tree = xmlReadMemory(NULL, 0, NULL, NULL, 0); + xmlFreeDoc(tree); + xmlCleanupParser(); + return 0; + }" + . auto/feature + + if [ $njs_found = no ]; then + + # FreeBSD port + + njs_feature="libxml2 in /usr/local/" + njs_feature_incs="/usr/local/include/libxml2 /usr/local/include" + njs_feature_libs="-L/usr/local/lib -lxml2" + + . auto/feature + fi + + if [ $njs_found = no ]; then + + # NetBSD port + + njs_feature="libxml2 in /usr/pkg/" + njs_feature_incs="/usr/pkg/include/libxml2 /usr/pkg/include" + njs_feature_libs="-L/usr/pkg/lib -lxml2" + + . auto/feature + fi + + if [ $njs_found = no ]; then + + # MacPorts + + njs_feature="libxml2 in /opt/local/" + njs_feature_incs="/opt/local/include/libxml2 /opt/local/include" + njs_feature_libs="-L/opt/local/lib -lxml2 -lxslt" + + . auto/feature + fi + + if [ $njs_found = yes ]; then + njs_feature="libxml2 version" + njs_feature_name=NJS_LIBXML2_VERSION + njs_feature_run=value + njs_feature_test="#include + #include + + int main() { + printf(\"\\\"%s\\\"\", LIBXML_DOTTED_VERSION); + return 0; + }" + . auto/feature + + NJS_HAVE_LIBXML2=YES + NJS_LIB_INCS="$NJS_LIB_INCS $njs_feature_incs" + NJS_LIB_AUX_LIBS="$NJS_LIB_AUX_LIBS $njs_feature_libs" + fi +fi diff -r 742347841e34 -r 99b9f83e4d4d auto/modules --- a/auto/modules Fri Jan 20 20:15:03 2023 -0800 +++ b/auto/modules Wed Jan 25 21:54:47 2023 -0800 @@ -21,6 +21,14 @@ if [ $NJS_OPENSSL = YES -a $NJS_HAVE_OPE . auto/module fi +if [ $NJS_LIBXML2 = YES -a $NJS_HAVE_LIBXML2 = YES ]; then + njs_module_name=njs_xml_module + njs_module_incs= + njs_module_srcs=external/njs_xml_module.c + + . auto/module +fi + njs_module_name=njs_fs_module njs_module_incs= njs_module_srcs=external/njs_fs_module.c diff -r 742347841e34 -r 99b9f83e4d4d auto/options --- a/auto/options Fri Jan 20 20:15:03 2023 -0800 +++ b/auto/options Wed Jan 25 21:54:47 2023 -0800 @@ -16,6 +16,7 @@ NJS_ADDR2LINE=NO NJS_TEST262=YES NJS_OPENSSL=YES +NJS_LIBXML2=YES NJS_PCRE=YES NJS_TRY_PCRE2=YES @@ -48,6 +49,7 @@ do --test262=*) NJS_TEST262="$value" ;; --no-openssl) NJS_OPENSSL=NO ;; + --no-libxml2) NJS_LIBXML2=NO ;; --no-pcre) NJS_PCRE=NO ;; --no-pcre2) NJS_TRY_PCRE2=NO ;; diff -r 742347841e34 -r 99b9f83e4d4d configure --- a/configure Fri Jan 20 20:15:03 2023 -0800 +++ b/configure Wed Jan 25 21:54:47 2023 -0800 @@ -51,6 +51,7 @@ NJS_LIB_AUX_LIBS= . auto/pcre . auto/readline . auto/openssl +. auto/libxml2 . auto/libbfd . auto/link diff -r 742347841e34 -r 99b9f83e4d4d external/njs_xml_module.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/external/njs_xml_module.c Wed Jan 25 21:54:47 2023 -0800 @@ -0,0 +1,1322 @@ + +/* + * Copyright (C) Dmitry Volyntsev + * Copyright (C) NGINX, Inc. + */ + + +#include +#include +#include +#include +#include +#include + + +typedef struct { + xmlDoc *doc; + xmlParserCtxt *ctx; +} njs_xml_doc_t; + + +typedef enum { + XML_NSET_TREE = 0, + XML_NSET_TREE_NO_COMMENTS, + XML_NSET_TREE_INVERT, +} njs_xml_nset_type_t; + + +typedef struct njs_xml_nset_s njs_xml_nset_t; + +struct njs_xml_nset_s { + xmlNodeSet *nodes; + xmlDoc *doc; + njs_xml_nset_type_t type; + njs_xml_nset_t *next; + njs_xml_nset_t *prev; +}; + +static njs_int_t njs_xml_ext_parse(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t unused); +static njs_int_t njs_xml_ext_canonicalization(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t unused); +static njs_int_t njs_xml_doc_ext_prop_keys(njs_vm_t *vm, njs_value_t *value, + njs_value_t *keys); +static njs_int_t njs_xml_doc_ext_root(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *unused, njs_value_t *retval); +static void njs_xml_doc_cleanup(void *data); +static njs_int_t njs_xml_node_ext_prop_keys(njs_vm_t *vm, njs_value_t *value, + njs_value_t *keys); +static njs_int_t njs_xml_node_ext_prop_handler(njs_vm_t *vm, + njs_object_prop_t *prop, njs_value_t *value, njs_value_t *unused, + njs_value_t *retval); +static njs_int_t njs_xml_attr_ext_prop_keys(njs_vm_t *vm, njs_value_t *value, + njs_value_t *keys); +static njs_int_t njs_xml_attr_ext_prop_handler(njs_vm_t *vm, + njs_object_prop_t *prop, njs_value_t *value, njs_value_t *unused, + njs_value_t *retval); +static njs_int_t njs_xml_node_ext_attrs(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *setval, njs_value_t *retval); +static njs_int_t njs_xml_node_ext_name(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *setval, njs_value_t *retval); +static njs_int_t njs_xml_node_ext_ns(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *setval, njs_value_t *retval); +static njs_int_t njs_xml_node_ext_parent(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *setval, njs_value_t *retval); +static njs_int_t njs_xml_node_ext_tags(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *setval, njs_value_t *retval); +static njs_int_t njs_xml_node_ext_text(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *setval, njs_value_t *retval); + +static njs_xml_nset_t *njs_xml_nset_create(njs_vm_t *vm, xmlDoc *doc, + xmlNodeSet *nodes, njs_xml_nset_type_t type); +static njs_xml_nset_t *njs_xml_nset_children(njs_vm_t *vm, xmlNode *parent); +static njs_xml_nset_t *njs_xml_nset_add(njs_xml_nset_t *nset, + njs_xml_nset_t *add); +static void njs_xml_nset_cleanup(void *data); +static void njs_xml_error(njs_vm_t *vm, njs_xml_doc_t *tree, const char *fmt, + ...); +static njs_int_t njs_xml_init(njs_vm_t *vm); + + +static njs_external_t njs_ext_xml[] = { + + { + .flags = NJS_EXTERN_PROPERTY | NJS_EXTERN_SYMBOL, + .name.symbol = NJS_SYMBOL_TO_STRING_TAG, + .u.property = { + .value = "xml", + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("parse"), + .writable = 1, + .configurable = 1, + .u.method = { + .native = njs_xml_ext_parse, + .magic8 = 0, + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("c14n"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = njs_xml_ext_canonicalization, + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("exclusiveC14n"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = njs_xml_ext_canonicalization, + .magic8 = 1, + } + }, + +}; + + +static njs_external_t njs_ext_xml_doc[] = { + + { + .flags = NJS_EXTERN_PROPERTY | NJS_EXTERN_SYMBOL, + .name.symbol = NJS_SYMBOL_TO_STRING_TAG, + .u.property = { + .value = "XMLDoc", + } + }, + + { + .flags = NJS_EXTERN_SELF, + .u.object = { + .enumerable = 1, + .prop_handler = njs_xml_doc_ext_root, + .keys = njs_xml_doc_ext_prop_keys, + } + }, + + { + .flags = NJS_EXTERN_PROPERTY, + .name.string = njs_str("$root"), + .enumerable = 1, + .u.property = { + .handler = njs_xml_doc_ext_root, + .magic32 = 1, + } + }, + +}; + + +static njs_external_t njs_ext_xml_node[] = { + + { + .flags = NJS_EXTERN_PROPERTY | NJS_EXTERN_SYMBOL, + .name.symbol = NJS_SYMBOL_TO_STRING_TAG, + .u.property = { + .value = "XMLNode", + } + }, + + { + .flags = NJS_EXTERN_SELF, + .u.object = { + .enumerable = 1, + .prop_handler = njs_xml_node_ext_prop_handler, + .keys = njs_xml_node_ext_prop_keys, + } + }, + + { + .flags = NJS_EXTERN_PROPERTY, + .name.string = njs_str("$attrs"), + .enumerable = 1, + .u.property = { + .handler = njs_xml_node_ext_attrs, + } + }, + + { + .flags = NJS_EXTERN_PROPERTY, + .name.string = njs_str("$name"), + .enumerable = 1, + .u.property = { + .handler = njs_xml_node_ext_name, + } + }, + + { + .flags = NJS_EXTERN_PROPERTY, + .name.string = njs_str("$ns"), + .enumerable = 1, + .u.property = { + .handler = njs_xml_node_ext_ns, + } + }, + + { + .flags = NJS_EXTERN_PROPERTY, + .name.string = njs_str("$parent"), + .enumerable = 1, + .u.property = { + .handler = njs_xml_node_ext_parent, + } + }, + + { + .flags = NJS_EXTERN_PROPERTY, + .name.string = njs_str("$tags"), + .enumerable = 1, + .u.property = { + .handler = njs_xml_node_ext_tags, + } + }, + + { + .flags = NJS_EXTERN_PROPERTY, + .name.string = njs_str("$text"), + .enumerable = 1, + .u.property = { + .handler = njs_xml_node_ext_text, + } + }, + +}; + + +static njs_external_t njs_ext_xml_attr[] = { + + { + .flags = NJS_EXTERN_PROPERTY | NJS_EXTERN_SYMBOL, + .name.symbol = NJS_SYMBOL_TO_STRING_TAG, + .u.property = { + .value = "XMLAttr", + } + }, + + { + .flags = NJS_EXTERN_SELF, + .u.object = { + .enumerable = 1, + .prop_handler = njs_xml_attr_ext_prop_handler, + .keys = njs_xml_attr_ext_prop_keys, + } + }, + +}; + + +njs_module_t njs_xml_module = { + .name = njs_str("xml"), + .init = njs_xml_init, +}; + + +static njs_int_t njs_xml_doc_proto_id; +static njs_int_t njs_xml_node_proto_id; +static njs_int_t njs_xml_attr_proto_id; + + +static njs_int_t +njs_xml_ext_parse(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, + njs_index_t unused) +{ + njs_int_t ret; + njs_str_t data; + njs_xml_doc_t *tree; + njs_mp_cleanup_t *cln; + + ret = njs_vm_value_to_bytes(vm, &data, njs_arg(args, nargs, 1)); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + tree = njs_mp_zalloc(njs_vm_memory_pool(vm), sizeof(njs_xml_doc_t)); + if (njs_slow_path(tree == NULL)) { + njs_vm_memory_error(vm); + return NJS_ERROR; + } + + tree->ctx = xmlNewParserCtxt(); + if (njs_slow_path(tree->ctx == NULL)) { + njs_vm_error(vm, "xmlNewParserCtxt() failed"); + return NJS_ERROR; + } + + tree->doc = xmlCtxtReadMemory(tree->ctx, (char *) data.start, data.length, + NULL, NULL, XML_PARSE_DTDVALID + | XML_PARSE_NOWARNING + | XML_PARSE_NOERROR); + if (njs_slow_path(tree->doc == NULL)) { + njs_xml_error(vm, tree, "failed to parse XML"); + return NJS_ERROR; + } + + cln = njs_mp_cleanup_add(njs_vm_memory_pool(vm), 0); + if (njs_slow_path(cln == NULL)) { + njs_vm_memory_error(vm); + return NJS_ERROR; + } + + cln->handler = njs_xml_doc_cleanup; + cln->data = tree; + + return njs_vm_external_create(vm, njs_vm_retval(vm), njs_xml_doc_proto_id, + tree, 0); +} + + +static njs_int_t +njs_xml_doc_ext_prop_keys(njs_vm_t *vm, njs_value_t *value, njs_value_t *keys) +{ + xmlNode *node; + njs_int_t ret; + njs_value_t *push; + njs_xml_doc_t *tree; + + tree = njs_vm_external(vm, njs_xml_doc_proto_id, value); + if (njs_slow_path(tree == NULL)) { + njs_value_undefined_set(keys); + return NJS_DECLINED; + } + + ret = njs_vm_array_alloc(vm, keys, 2); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + for (node = xmlDocGetRootElement(tree->doc); + node != NULL; + node = node->next) + { + if (node->type != XML_ELEMENT_NODE) { + continue; + } + + push = njs_vm_array_push(vm, keys); + if (njs_slow_path(push == NULL)) { + return NJS_ERROR; + } + + ret = njs_vm_value_string_create(vm, push, node->name, + njs_strlen(node->name)); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + } + + return NJS_OK; +} + + +static njs_int_t +njs_xml_doc_ext_root(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, + njs_value_t *unused, njs_value_t *retval) +{ + xmlNode *node; + njs_int_t ret; + njs_str_t name; + njs_bool_t any; + njs_xml_doc_t *tree; + + tree = njs_vm_external(vm, njs_xml_doc_proto_id, value); + if (njs_slow_path(tree == NULL)) { + njs_value_undefined_set(retval); + return NJS_DECLINED; + } + + any = njs_vm_prop_magic32(prop); + + if (!any) { + ret = njs_vm_prop_name(vm, prop, &name); + if (njs_slow_path(ret != NJS_OK)) { + njs_value_undefined_set(retval); + return NJS_DECLINED; + } + } + + for (node = xmlDocGetRootElement(tree->doc); + node != NULL; + node = node->next) + { + if (node->type != XML_ELEMENT_NODE) { + continue; + } + + if (!any) { + if (name.length != njs_strlen(node->name) + || njs_strncmp(name.start, node->name, name.length) != 0) + { + continue; + } + } + + return njs_vm_external_create(vm, retval, njs_xml_node_proto_id, node, + 0); + } + + njs_value_undefined_set(retval); + + return NJS_DECLINED; +} + + +static void +njs_xml_doc_cleanup(void *data) +{ + njs_xml_doc_t *current = data; + + xmlFreeDoc(current->doc); + xmlFreeParserCtxt(current->ctx); +} + + +static njs_int_t +njs_xml_node_ext_prop_keys(njs_vm_t *vm, njs_value_t *value, njs_value_t *keys) +{ + xmlNode *node, *current; + njs_int_t ret; + njs_value_t *push; + + current = njs_vm_external(vm, njs_xml_node_proto_id, value); + if (njs_slow_path(current == NULL)) { + njs_value_undefined_set(keys); + return NJS_DECLINED; + } + + ret = njs_vm_array_alloc(vm, keys, 2); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + if (current->name != NULL && current->type == XML_ELEMENT_NODE) { + push = njs_vm_array_push(vm, keys); + if (njs_slow_path(push == NULL)) { + return NJS_ERROR; + } + + ret = njs_vm_value_string_set(vm, push, (u_char *) "$name", + njs_length("$name")); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + } + + if (current->ns != NULL) { + push = njs_vm_array_push(vm, keys); + if (njs_slow_path(push == NULL)) { + return NJS_ERROR; + } + + ret = njs_vm_value_string_set(vm, push, (u_char *) "$ns", + njs_length("$ns")); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + } + + if (current->properties != NULL) { + push = njs_vm_array_push(vm, keys); + if (njs_slow_path(push == NULL)) { + return NJS_ERROR; + } + + ret = njs_vm_value_string_set(vm, push, (u_char *) "$attrs", + njs_length("$attrs")); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + } + + if (current->children != NULL && current->children->content != NULL) { + push = njs_vm_array_push(vm, keys); + if (njs_slow_path(push == NULL)) { + return NJS_ERROR; + } + + ret = njs_vm_value_string_set(vm, push, (u_char *) "$text", + njs_length("$text")); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + } + + for (node = current->children; node != NULL; node = node->next) { + if (node->type != XML_ELEMENT_NODE) { + continue; + } + + push = njs_vm_array_push(vm, keys); + if (njs_slow_path(push == NULL)) { + return NJS_ERROR; + } + + ret = njs_vm_value_string_set(vm, push, (u_char *) "$tags", + njs_length("$tags")); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + break; + } + + return NJS_OK; +} + + +static njs_int_t +njs_xml_node_ext_prop_handler(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *unused, njs_value_t *retval) +{ + size_t size; + xmlAttr *attr; + xmlNode *node, *current; + njs_int_t ret; + njs_str_t name; + njs_value_t *push; + const u_char *content; + + /* + * $tag$foo - the first tag child with the name "foo" + * $tags$foo - the all children with the name "foo" as an array + * $attr$foo - the attribute with the name "foo" + * foo - the same as $tag$foo + */ + + current = njs_vm_external(vm, njs_xml_node_proto_id, value); + if (njs_slow_path(current == NULL)) { + njs_value_undefined_set(retval); + return NJS_DECLINED; + } + + ret = njs_vm_prop_name(vm, prop, &name); + if (njs_slow_path(ret != NJS_OK)) { + njs_value_undefined_set(retval); + return NJS_DECLINED; + } + + if (name.length > 1 && name.start[0] == '$') { + if (name.length > njs_length("$attr$") + && njs_strncmp(&name.start[1], "attr$", njs_length("attr$")) == 0) + { + for (attr = current->properties; attr != NULL; attr = attr->next) { + if (attr->type != XML_ATTRIBUTE_NODE) { + continue; + } + + size = njs_strlen(attr->name); + + if (name.length != (size + njs_length("$attr$")) + || njs_strncmp(&name.start[njs_length("$attr$")], + attr->name, size) != 0) + { + continue; + } + + content = (const u_char *) attr->children->content; + + return njs_vm_value_string_create(vm, retval, content, + njs_strlen(content)); + } + } + + if (name.length > njs_length("$tag$") + && njs_strncmp(&name.start[1], "tag$", njs_length("tag$")) == 0) + { + for (node = current->children; node != NULL; node = node->next) { + if (node->type != XML_ELEMENT_NODE) { + continue; + } + + size = njs_strlen(node->name); + + if (name.length != (size + njs_length("$tag$")) + || njs_strncmp(&name.start[njs_length("$tag$")], + node->name, size) != 0) + { + continue; + } + + return njs_vm_external_create(vm, retval, njs_xml_node_proto_id, + node, 0); + } + } + + if (name.length >= njs_length("$tags$") + && njs_strncmp(&name.start[1], "tags$", njs_length("tags$")) == 0) + { + ret = njs_vm_array_alloc(vm, retval, 2); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + for (node = current->children; node != NULL; node = node->next) { + if (node->type != XML_ELEMENT_NODE) { + continue; + } + + size = njs_strlen(node->name); + + if (name.length > njs_length("$tags$") + && (name.length != (size + njs_length("$tags$")) + || njs_strncmp(&name.start[njs_length("$tags$")], + node->name, size) != 0)) + { + continue; + } + + push = njs_vm_array_push(vm, retval); + if (njs_slow_path(push == NULL)) { + return NJS_ERROR; + } + + ret = njs_vm_external_create(vm, push, njs_xml_node_proto_id, + node, 0); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + } + + return NJS_OK; + } + } + + for (node = current->children; node != NULL; node = node->next) { + if (node->type != XML_ELEMENT_NODE) { + continue; + } + + size = njs_strlen(node->name); + + if (name.length != size + || njs_strncmp(name.start, node->name, size) != 0) + { + continue; + } + + return njs_vm_external_create(vm, retval, njs_xml_node_proto_id, + node, 0); + } + + njs_value_undefined_set(retval); + + return NJS_DECLINED; +} + + +static njs_int_t +njs_xml_node_ext_attrs(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *setval, njs_value_t *retval) +{ + xmlNode *current; + + current = njs_vm_external(vm, njs_xml_node_proto_id, value); + if (njs_slow_path(current == NULL || current->properties == NULL)) { + njs_value_undefined_set(retval); + return NJS_DECLINED; + } + + return njs_vm_external_create(vm, retval, njs_xml_attr_proto_id, + current->properties, 0); +} + + +static njs_int_t +njs_xml_node_ext_name(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, + njs_value_t *setval, njs_value_t *retval) +{ + xmlNode *current; + + current = njs_vm_external(vm, njs_xml_node_proto_id, value); + if (current == NULL || current->type != XML_ELEMENT_NODE) { + njs_value_undefined_set(retval); + return NJS_DECLINED; + } + + return njs_vm_value_string_create(vm, retval, current->name, + njs_strlen(current->name)); +} + + +static njs_int_t +njs_xml_node_ext_ns(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *setval, njs_value_t *retval) +{ + xmlNode *current; + + current = njs_vm_external(vm, njs_xml_node_proto_id, value); + if (njs_slow_path(current == NULL || current->ns == NULL)) { + njs_value_undefined_set(retval); + return NJS_DECLINED; + } + + return njs_vm_value_string_create(vm, retval, current->ns->href, + njs_strlen(current->ns->href)); +} + + +static njs_int_t +njs_xml_node_ext_parent(njs_vm_t *vm, njs_object_prop_t *prop, + njs_value_t *value, njs_value_t *setval, njs_value_t *retval) +{ + xmlNode *current; + + current = njs_vm_external(vm, njs_xml_node_proto_id, value); + if (njs_slow_path(current == NULL + || current->parent == NULL + || current->parent->type != XML_ELEMENT_NODE)) + { + njs_value_undefined_set(retval); + return NJS_DECLINED; + } + + return njs_vm_external_create(vm, retval, njs_xml_node_proto_id, + current->parent, 0); +} + + +static njs_int_t +njs_xml_node_ext_tags(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, + njs_value_t *setval, njs_value_t *retval) +{ + xmlNode *node, *current; + njs_int_t ret; + njs_value_t *push; + + current = njs_vm_external(vm, njs_xml_node_proto_id, value); + if (njs_slow_path(current == NULL || current->children == NULL)) { + njs_value_undefined_set(retval); + return NJS_DECLINED; + } + + ret = njs_vm_array_alloc(vm, retval, 2); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + for (node = current->children; node != NULL; node = node->next) { + if (node->type != XML_ELEMENT_NODE) { + continue; + } + + push = njs_vm_array_push(vm, retval); + if (njs_slow_path(push == NULL)) { + return NJS_ERROR; + } + + ret = njs_vm_external_create(vm, push, njs_xml_node_proto_id, node, 0); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + } + + return NJS_OK; +} + + +static njs_int_t +njs_xml_node_ext_text(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, + njs_value_t *setval, njs_value_t *retval) +{ + xmlNode *current, *node; + njs_int_t ret; + njs_chb_t chain; + + current = njs_vm_external(vm, njs_xml_node_proto_id, value); + if (njs_slow_path(current == NULL)) { + njs_value_undefined_set(retval); + return NJS_DECLINED; + } + + njs_chb_init(&chain, njs_vm_memory_pool(vm)); + + for (node = current->children; node != NULL; node = node->next) { + if (node->type != XML_TEXT_NODE) { + continue; + } + + njs_chb_append(&chain, node->content, njs_strlen(node->content)); + } + + ret = njs_vm_value_string_create_chb(vm, retval, &chain); + + njs_chb_destroy(&chain); + + return ret; +} + + +static int +njs_xml_buf_write_cb(void *context, const char *buffer, int len) +{ + njs_chb_t *chain = context; + + njs_chb_append(chain, buffer, len); + + return chain->error ? -1 : len; +} + + +static int +njs_xml_node_one_contains(njs_xml_nset_t *nset, xmlNode *node, xmlNode *parent) +{ + int in; + xmlNs ns; + + if (nset->type == XML_NSET_TREE_NO_COMMENTS + && node->type == XML_COMMENT_NODE) + { + return 0; + } + + in = 1; + + if (nset->nodes != NULL) { + if (node->type != XML_NAMESPACE_DECL) { + in = xmlXPathNodeSetContains(nset->nodes, node); + + } else { + + memcpy(&ns, node, sizeof(ns)); + + /* libxml2 workaround, check xpath.c for details */ + + if ((parent != NULL) && (parent->type == XML_ATTRIBUTE_NODE)) { + ns.next = (xmlNs *) parent->parent; + + } else { + ns.next = (xmlNs *) parent; + } + + in = xmlXPathNodeSetContains(nset->nodes, (xmlNode *) &ns); + } + } + + switch (nset->type) { + case XML_NSET_TREE: + case XML_NSET_TREE_NO_COMMENTS: + if (in != 0) { + return 1; + } + + if ((parent != NULL) && (parent->type == XML_ELEMENT_NODE)) { + return njs_xml_node_one_contains(nset, parent, parent->parent); + } + + return 0; + + case XML_NSET_TREE_INVERT: + default: + if (in != 0) { + return 0; + } + + if ((parent != NULL) && (parent->type == XML_ELEMENT_NODE)) { + return njs_xml_node_one_contains(nset, parent, parent->parent); + } + } From arut at nginx.com Thu Jan 26 11:50:44 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Thu, 26 Jan 2023 15:50:44 +0400 Subject: [PATCH 0 of 3] Directives for enabling http2 and http3 Message-ID: Hi, These patches change http2 and http3 configurations. Previously, both protocols were configured by "listen" parameters: listen 8800 ssl http2; listen 8443 http3; The patches introduce new directives for configuring http2 and http3 in server scope: server { listen 8800 ssl; listen 8443 quic; http2 on; http3 on; ... } Now protocol list is a property of server, not a listener. After choosing a server by SNI, nginx has a list of protocols supported in that particular server. The right protocol is then chosen in ALPN callback. For plain http2 connections, a simple preread is implemented which tells http/1 from http2 by the first symbol similar to TLS signature preread in http. -- Roman Arutyunyan From arut at nginx.com Thu Jan 26 11:50:45 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Thu, 26 Jan 2023 15:50:45 +0400 Subject: [PATCH 1 of 3] HTTP/3: "quic" parameter of "listen" directive In-Reply-To: References: Message-ID: <250539ea3e05d0ac2ead.1674733845@arut-laptop> # HG changeset patch # User Roman Arutyunyan # Date 1674653899 -14400 # Wed Jan 25 17:38:19 2023 +0400 # Branch quic # Node ID 250539ea3e05d0ac2ead4784a1b44f28ff4d1ef6 # Parent a954b551dc3f8f059140e5843c49341cae55b98b HTTP/3: "quic" parameter of "listen" directive. Now "listen" directve has a new "quic" parameter which enables QUIC protocol for the address. Further, to enable HTTP/3, a new directive "http3" is introduced. The hq-interop protocol is enabled by "http3_hq" as before. Now application protocol is chosen by ALPN. Previously used "http3" parameter of "listen" is eliminated. diff --git a/README b/README --- a/README +++ b/README @@ -93,13 +93,13 @@ 2. Installing 3. Configuration - The HTTP "listen" directive got a new option "http3" which enables - HTTP/3 over QUIC on the specified port. + The HTTP "listen" directive got a new option "quic" which enables + QUIC as client transport protocol instead of TCP. The Stream "listen" directive got a new option "quic" which enables QUIC as client transport protocol instead of TCP or plain UDP. - Along with "http3" or "quic", it's also possible to specify "reuseport" + Along with "quic", it's also possible to specify "reuseport" option [8] to make it work properly with multiple workers. To enable address validation: @@ -133,12 +133,13 @@ 3. Configuration A number of directives were added that configure HTTP/3: + http3 + http3_hq http3_stream_buffer_size http3_max_concurrent_pushes http3_max_concurrent_streams http3_push http3_push_preload - http3_hq (requires NGX_HTTP_V3_HQ macro) In http, an additional variable is available: $http3. The value of $http3 is "h3" for HTTP/3 connections, @@ -160,13 +161,15 @@ Example configuration: server { # for better compatibility it's recommended # to use the same port for quic and https - listen 8443 http3 reuseport; + listen 8443 quic reuseport; listen 8443 ssl; ssl_certificate certs/example.com.crt; ssl_certificate_key certs/example.com.key; ssl_protocols TLSv1.3; + http3 on; + location / { # required for browsers to direct them into quic port add_header Alt-Svc 'h3=":8443"; ma=86400'; diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -427,7 +427,7 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t #if (NGX_HTTP_V2 || NGX_HTTP_V3) ngx_http_connection_t *hc; #endif -#if (NGX_HTTP_V3 && NGX_HTTP_V3_HQ) +#if (NGX_HTTP_V3) ngx_http_v3_srv_conf_t *h3scf; #endif #if (NGX_HTTP_V2 || NGX_HTTP_V3 || NGX_DEBUG) @@ -455,19 +455,26 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t } else #endif #if (NGX_HTTP_V3) - if (hc->addr_conf->http3) { + if (hc->addr_conf->quic) { -#if (NGX_HTTP_V3_HQ) h3scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v3_module); - if (h3scf->hq) { + if (h3scf->enable && h3scf->enable_hq) { + srv = (unsigned char *) NGX_HTTP_V3_ALPN_PROTO + NGX_HTTP_V3_HQ_ALPN_PROTO; + srvlen = sizeof(NGX_HTTP_V3_ALPN_PROTO NGX_HTTP_V3_HQ_ALPN_PROTO) + - 1; + + } else if (h3scf->enable) { + srv = (unsigned char *) NGX_HTTP_V3_ALPN_PROTO; + srvlen = sizeof(NGX_HTTP_V3_ALPN_PROTO) - 1; + + } else if (h3scf->enable_hq) { srv = (unsigned char *) NGX_HTTP_V3_HQ_ALPN_PROTO; srvlen = sizeof(NGX_HTTP_V3_HQ_ALPN_PROTO) - 1; - } else -#endif - { - srv = (unsigned char *) NGX_HTTP_V3_ALPN_PROTO; - srvlen = sizeof(NGX_HTTP_V3_ALPN_PROTO) - 1; + + } else { + return SSL_TLSEXT_ERR_ALERT_FATAL; } } else @@ -1313,12 +1320,12 @@ ngx_http_ssl_init(ngx_conf_t *cf) addr = port[p].addrs.elts; for (a = 0; a < port[p].addrs.nelts; a++) { - if (!addr[a].opt.ssl && !addr[a].opt.http3) { + if (!addr[a].opt.ssl && !addr[a].opt.quic) { continue; } - if (addr[a].opt.http3) { - name = "http3"; + if (addr[a].opt.quic) { + name = "quic"; } else { name = "ssl"; @@ -1329,7 +1336,7 @@ ngx_http_ssl_init(ngx_conf_t *cf) if (sscf->certificates) { - if (addr[a].opt.http3 && !(sscf->protocols & NGX_SSL_TLSv1_3)) { + if (addr[a].opt.quic && !(sscf->protocols & NGX_SSL_TLSv1_3)) { ngx_log_error(NGX_LOG_EMERG, cf->log, 0, "\"ssl_protocols\" must enable TLSv1.3 for " "the \"listen ... %s\" directive in %s:%ui", diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c --- a/src/http/ngx_http.c +++ b/src/http/ngx_http.c @@ -1241,7 +1241,7 @@ ngx_http_add_addresses(ngx_conf_t *cf, n ngx_uint_t http2; #endif #if (NGX_HTTP_V3) - ngx_uint_t http3; + ngx_uint_t quic; #endif /* @@ -1279,7 +1279,7 @@ ngx_http_add_addresses(ngx_conf_t *cf, n http2 = lsopt->http2 || addr[i].opt.http2; #endif #if (NGX_HTTP_V3) - http3 = lsopt->http3 || addr[i].opt.http3; + quic = lsopt->quic || addr[i].opt.quic; #endif if (lsopt->set) { @@ -1318,7 +1318,7 @@ ngx_http_add_addresses(ngx_conf_t *cf, n addr[i].opt.http2 = http2; #endif #if (NGX_HTTP_V3) - addr[i].opt.http3 = http3; + addr[i].opt.quic = quic; #endif return NGX_OK; @@ -1823,7 +1823,7 @@ ngx_http_add_listening(ngx_conf_t *cf, n #if (NGX_HTTP_V3) - ls->quic = addr->opt.http3; + ls->quic = addr->opt.quic; if (ls->quic) { ngx_rbtree_init(&ls->rbtree, &ls->sentinel, @@ -1865,7 +1865,7 @@ ngx_http_add_addrs(ngx_conf_t *cf, ngx_h addrs[i].conf.http2 = addr[i].opt.http2; #endif #if (NGX_HTTP_V3) - addrs[i].conf.http3 = addr[i].opt.http3; + addrs[i].conf.quic = addr[i].opt.quic; #endif addrs[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; @@ -1933,7 +1933,7 @@ ngx_http_add_addrs6(ngx_conf_t *cf, ngx_ addrs6[i].conf.http2 = addr[i].opt.http2; #endif #if (NGX_HTTP_V3) - addrs6[i].conf.http3 = addr[i].opt.http3; + addrs6[i].conf.quic = addr[i].opt.quic; #endif addrs6[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -4189,14 +4189,14 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx #endif } - if (ngx_strcmp(value[n].data, "http3") == 0) { + if (ngx_strcmp(value[n].data, "quic") == 0) { #if (NGX_HTTP_V3) - lsopt.http3 = 1; + lsopt.quic = 1; lsopt.type = SOCK_DGRAM; continue; #else ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "the \"http3\" parameter requires " + "the \"quic\" parameter requires " "ngx_http_v3_module"); return NGX_CONF_ERROR; #endif @@ -4304,8 +4304,8 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx } #if (NGX_HTTP_SSL && NGX_HTTP_V3) - if (lsopt.ssl && lsopt.http3) { - return "\"ssl\" parameter is incompatible with \"http3\""; + if (lsopt.ssl && lsopt.quic) { + return "\"ssl\" parameter is incompatible with \"quic\""; } #endif diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -75,7 +75,7 @@ typedef struct { unsigned wildcard:1; unsigned ssl:1; unsigned http2:1; - unsigned http3:1; + unsigned quic:1; #if (NGX_HAVE_INET6) unsigned ipv6only:1; #endif @@ -239,7 +239,7 @@ struct ngx_http_addr_conf_s { unsigned ssl:1; unsigned http2:1; - unsigned http3:1; + unsigned quic:1; unsigned proxy_protocol:1; }; diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -325,7 +325,7 @@ ngx_http_init_connection(ngx_connection_ #endif #if (NGX_HTTP_V3) - if (hc->addr_conf->http3) { + if (hc->addr_conf->quic) { ngx_http_v3_init_stream(c); return; } diff --git a/src/http/v3/ngx_http_v3.c b/src/http/v3/ngx_http_v3.c --- a/src/http/v3/ngx_http_v3.c +++ b/src/http/v3/ngx_http_v3.c @@ -17,12 +17,9 @@ static void ngx_http_v3_cleanup_session( ngx_int_t ngx_http_v3_init_session(ngx_connection_t *c) { - ngx_pool_cleanup_t *cln; - ngx_http_connection_t *hc; - ngx_http_v3_session_t *h3c; -#if (NGX_HTTP_V3_HQ) - ngx_http_v3_srv_conf_t *h3scf; -#endif + ngx_pool_cleanup_t *cln; + ngx_http_connection_t *hc; + ngx_http_v3_session_t *h3c; hc = c->data; @@ -36,13 +33,6 @@ ngx_http_v3_init_session(ngx_connection_ h3c->max_push_id = (uint64_t) -1; h3c->goaway_push_id = (uint64_t) -1; -#if (NGX_HTTP_V3_HQ) - h3scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v3_module); - if (h3scf->hq) { - h3c->hq = 1; - } -#endif - ngx_queue_init(&h3c->blocked); ngx_queue_init(&h3c->pushing); diff --git a/src/http/v3/ngx_http_v3.h b/src/http/v3/ngx_http_v3.h --- a/src/http/v3/ngx_http_v3.h +++ b/src/http/v3/ngx_http_v3.h @@ -21,6 +21,7 @@ #define NGX_HTTP_V3_ALPN_PROTO "\x02h3" #define NGX_HTTP_V3_HQ_ALPN_PROTO "\x0Ahq-interop" +#define NGX_HTTP_V3_HQ_PROTO "hq-interop" #define NGX_HTTP_V3_VARLEN_INT_LEN 4 #define NGX_HTTP_V3_PREFIX_INT_LEN 11 @@ -101,13 +102,12 @@ typedef struct { + ngx_flag_t enable; + ngx_flag_t enable_hq; size_t max_table_capacity; ngx_uint_t max_blocked_streams; ngx_uint_t max_concurrent_pushes; ngx_uint_t max_concurrent_streams; -#if (NGX_HTTP_V3_HQ) - ngx_flag_t hq; -#endif ngx_quic_conf_t quic; } ngx_http_v3_srv_conf_t; @@ -147,9 +147,7 @@ struct ngx_http_v3_session_s { off_t payload_bytes; unsigned goaway:1; -#if (NGX_HTTP_V3_HQ) unsigned hq:1; -#endif ngx_connection_t *known_streams[NGX_HTTP_V3_MAX_KNOWN_STREAM]; }; diff --git a/src/http/v3/ngx_http_v3_module.c b/src/http/v3/ngx_http_v3_module.c --- a/src/http/v3/ngx_http_v3_module.c +++ b/src/http/v3/ngx_http_v3_module.c @@ -32,6 +32,20 @@ static ngx_conf_post_t ngx_http_quic_mt static ngx_command_t ngx_http_v3_commands[] = { + { ngx_string("http3"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v3_srv_conf_t, enable), + NULL }, + + { ngx_string("http3_hq"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v3_srv_conf_t, enable_hq), + NULL }, + { ngx_string("http3_max_concurrent_pushes"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, @@ -46,15 +60,6 @@ static ngx_command_t ngx_http_v3_comman offsetof(ngx_http_v3_srv_conf_t, max_concurrent_streams), NULL }, -#if (NGX_HTTP_V3_HQ) - { ngx_string("http3_hq"), - NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, - ngx_conf_set_flag_slot, - NGX_HTTP_SRV_CONF_OFFSET, - offsetof(ngx_http_v3_srv_conf_t, hq), - NULL }, -#endif - { ngx_string("http3_push"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_http_v3_push, @@ -160,14 +165,12 @@ static ngx_int_t ngx_http_v3_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { - if (r->connection->quic) { -#if (NGX_HTTP_V3_HQ) + ngx_http_v3_session_t *h3c; - ngx_http_v3_srv_conf_t *h3scf; + if (r->connection->quic) { + h3c = ngx_http_v3_get_session(r->connection); - h3scf = ngx_http_get_module_srv_conf(r, ngx_http_v3_module); - - if (h3scf->hq) { + if (h3c->hq) { v->len = sizeof("hq") - 1; v->valid = 1; v->no_cacheable = 0; @@ -177,8 +180,6 @@ ngx_http_v3_variable(ngx_http_request_t return NGX_OK; } -#endif - v->len = sizeof("h3") - 1; v->valid = 1; v->no_cacheable = 0; @@ -232,12 +233,12 @@ ngx_http_v3_create_srv_conf(ngx_conf_t * * h3scf->quic.timeout = 0; * h3scf->max_blocked_streams = 0; */ + + h3scf->enable = NGX_CONF_UNSET; + h3scf->enable_hq = NGX_CONF_UNSET; h3scf->max_table_capacity = NGX_HTTP_V3_MAX_TABLE_CAPACITY; h3scf->max_concurrent_pushes = NGX_CONF_UNSET_UINT; h3scf->max_concurrent_streams = NGX_CONF_UNSET_UINT; -#if (NGX_HTTP_V3_HQ) - h3scf->hq = NGX_CONF_UNSET; -#endif h3scf->quic.mtu = NGX_CONF_UNSET_SIZE; h3scf->quic.stream_buffer_size = NGX_CONF_UNSET_SIZE; @@ -264,6 +265,10 @@ ngx_http_v3_merge_srv_conf(ngx_conf_t *c ngx_http_ssl_srv_conf_t *sscf; + ngx_conf_merge_value(conf->enable, prev->enable, 0); + + ngx_conf_merge_value(conf->enable_hq, prev->enable_hq, 0); + ngx_conf_merge_uint_value(conf->max_concurrent_pushes, prev->max_concurrent_pushes, 10); @@ -272,11 +277,6 @@ ngx_http_v3_merge_srv_conf(ngx_conf_t *c conf->max_blocked_streams = conf->max_concurrent_streams; -#if (NGX_HTTP_V3_HQ) - ngx_conf_merge_value(conf->hq, prev->hq, 0); -#endif - - ngx_conf_merge_size_value(conf->quic.mtu, prev->quic.mtu, NGX_QUIC_MAX_UDP_PAYLOAD_SIZE); diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -110,7 +110,10 @@ ngx_http_v3_init_stream(ngx_connection_t ngx_int_t ngx_http_v3_init(ngx_connection_t *c) { + unsigned int len; + const unsigned char *data; ngx_http_v3_session_t *h3c; + ngx_http_v3_srv_conf_t *h3scf; ngx_http_core_loc_conf_t *clcf; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 init"); @@ -119,11 +122,23 @@ ngx_http_v3_init(ngx_connection_t *c) clcf = ngx_http_v3_get_module_loc_conf(c, ngx_http_core_module); ngx_add_timer(&h3c->keepalive, clcf->keepalive_timeout); -#if (NGX_HTTP_V3_HQ) - if (h3c->hq) { - return NGX_OK; + h3scf = ngx_http_v3_get_module_srv_conf(c, ngx_http_v3_module); + + if (h3scf->enable_hq) { + if (!h3scf->enable) { + h3c->hq = 1; + return NGX_OK; + } + + SSL_get0_alpn_selected(c->ssl->connection, &data, &len); + + if (len == sizeof(NGX_HTTP_V3_HQ_PROTO) - 1 + && ngx_strncmp(data, NGX_HTTP_V3_HQ_PROTO, len) == 0) + { + h3c->hq = 1; + return NGX_OK; + } } -#endif return ngx_http_v3_send_settings(c); } @@ -147,10 +162,7 @@ ngx_http_v3_shutdown(ngx_connection_t *c if (!h3c->goaway) { h3c->goaway = 1; -#if (NGX_HTTP_V3_HQ) - if (!h3c->hq) -#endif - { + if (!h3c->hq) { (void) ngx_http_v3_send_goaway(c, h3c->next_request_id); } @@ -205,10 +217,7 @@ ngx_http_v3_init_request_stream(ngx_conn { h3c->goaway = 1; -#if (NGX_HTTP_V3_HQ) - if (!h3c->hq) -#endif - { + if (!h3c->hq) { if (ngx_http_v3_send_goaway(c, h3c->next_request_id) != NGX_OK) { ngx_http_close_connection(c); return; @@ -236,10 +245,7 @@ ngx_http_v3_init_request_stream(ngx_conn rev = c->read; -#if (NGX_HTTP_V3_HQ) - if (!h3c->hq) -#endif - { + if (!h3c->hq) { rev->handler = ngx_http_v3_wait_request_handler; c->write->handler = ngx_http_empty_handler; } @@ -398,14 +404,14 @@ ngx_http_v3_wait_request_handler(ngx_eve void ngx_http_v3_reset_stream(ngx_connection_t *c) { + ngx_http_v3_session_t *h3c; ngx_http_v3_srv_conf_t *h3scf; h3scf = ngx_http_v3_get_module_srv_conf(c, ngx_http_v3_module); - if (h3scf->max_table_capacity > 0 && !c->read->eof -#if (NGX_HTTP_V3_HQ) - && !h3scf->hq -#endif + h3c = ngx_http_v3_get_session(c); + + if (h3scf->max_table_capacity > 0 && !c->read->eof && !h3c->hq && (c->quic->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0) { (void) ngx_http_v3_send_cancel_stream(c, c->quic->id); diff --git a/src/http/v3/ngx_http_v3_uni.c b/src/http/v3/ngx_http_v3_uni.c --- a/src/http/v3/ngx_http_v3_uni.c +++ b/src/http/v3/ngx_http_v3_uni.c @@ -37,12 +37,9 @@ void ngx_http_v3_init_uni_stream(ngx_connection_t *c) { uint64_t n; -#if (NGX_HTTP_V3_HQ) ngx_http_v3_session_t *h3c; -#endif ngx_http_v3_uni_stream_t *us; -#if (NGX_HTTP_V3_HQ) h3c = ngx_http_v3_get_session(c); if (h3c->hq) { ngx_http_v3_finalize_connection(c, @@ -52,7 +49,6 @@ ngx_http_v3_init_uni_stream(ngx_connecti ngx_http_v3_close_uni_stream(c); return; } -#endif ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 init uni stream"); From arut at nginx.com Thu Jan 26 11:50:46 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Thu, 26 Jan 2023 15:50:46 +0400 Subject: [PATCH 2 of 3] HTTP/3: trigger more compatibility errors for "listen quic" In-Reply-To: References: Message-ID: <555913c358221f647bba.1674733846@arut-laptop> # HG changeset patch # User Roman Arutyunyan # Date 1674732333 -14400 # Thu Jan 26 15:25:33 2023 +0400 # Branch quic # Node ID 555913c358221f647bbace26165bef5eb614add4 # Parent 250539ea3e05d0ac2ead4784a1b44f28ff4d1ef6 HTTP/3: trigger more compatibility errors for "listen quic". Now "ssl", "proxy_protocol" and "http2" are not allowed with "quic" in "listen" directive. Previously, only "ssl" was not allowed. diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -4303,10 +4303,26 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx return NGX_CONF_ERROR; } -#if (NGX_HTTP_SSL && NGX_HTTP_V3) - if (lsopt.ssl && lsopt.quic) { - return "\"ssl\" parameter is incompatible with \"quic\""; - } +#if (NGX_HTTP_V3) + + if (lsopt.quic) { +#if (NGX_HTTP_SSL) + if (lsopt.ssl) { + return "\"ssl\" parameter is incompatible with \"quic\""; + } +#endif + +#if (NGX_HTTP_V2) + if (lsopt.http2) { + return "\"http2\" parameter is incompatible with \"quic\""; + } +#endif + + if (lsopt.proxy_protocol) { + return "\"proxy_protocol\" parameter is incompatible with \"quic\""; + } + } + #endif for (n = 0; n < u.naddrs; n++) { From arut at nginx.com Thu Jan 26 11:50:47 2023 From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=) Date: Thu, 26 Jan 2023 15:50:47 +0400 Subject: [PATCH 3 of 3] HTTP/2: "http2" directive In-Reply-To: References: Message-ID: <819737783463d7e38ea8.1674733847@arut-laptop> # HG changeset patch # User Roman Arutyunyan # Date 1674649725 -14400 # Wed Jan 25 16:28:45 2023 +0400 # Branch quic # Node ID 819737783463d7e38ea80109a976db1d3a9bb2db # Parent 555913c358221f647bbace26165bef5eb614add4 HTTP/2: "http2" directive. The directive enables HTTP/2 in the current server. The previous way to enable HTTP/2 via "listen ... http2" is now deprecated. diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -427,6 +427,9 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t #if (NGX_HTTP_V2 || NGX_HTTP_V3) ngx_http_connection_t *hc; #endif +#if (NGX_HTTP_V2) + ngx_http_v2_srv_conf_t *h2scf; +#endif #if (NGX_HTTP_V3) ngx_http_v3_srv_conf_t *h3scf; #endif @@ -448,12 +451,9 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t hc = c->data; #endif -#if (NGX_HTTP_V2) - if (hc->addr_conf->http2) { - srv = (unsigned char *) NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTOS; - srvlen = sizeof(NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTOS) - 1; - } else -#endif + srv = (unsigned char *) NGX_HTTP_ALPN_PROTOS; + srvlen = sizeof(NGX_HTTP_ALPN_PROTOS) - 1; + #if (NGX_HTTP_V3) if (hc->addr_conf->quic) { @@ -479,10 +479,16 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t } else #endif +#if (NGX_HTTP_V2) { - srv = (unsigned char *) NGX_HTTP_ALPN_PROTOS; - srvlen = sizeof(NGX_HTTP_ALPN_PROTOS) - 1; + h2scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v2_module); + + if (hc->addr_conf->http2 || h2scf->enable) { + srv = (unsigned char *) NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTOS; + srvlen = sizeof(NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTOS) - 1; + } } +#endif if (SSL_select_next_proto((unsigned char **) out, outlen, srv, srvlen, in, inlen) diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -4179,6 +4179,11 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx if (ngx_strcmp(value[n].data, "http2") == 0) { #if (NGX_HTTP_V2) + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "the \"listen ... http2\" directive " + "is deprecated, use " + "the \"http2\" directive instead"); + lsopt.http2 = 1; continue; #else diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -52,6 +52,10 @@ static u_char *ngx_http_log_error(ngx_lo static u_char *ngx_http_log_error_handler(ngx_http_request_t *r, ngx_http_request_t *sr, u_char *buf, size_t len); +#if (NGX_HTTP_V2) +static void ngx_http_try_v2(ngx_event_t *rev); +#endif + #if (NGX_HTTP_SSL) static void ngx_http_ssl_handshake(ngx_event_t *rev); static void ngx_http_ssl_handshake_handler(ngx_connection_t *c); @@ -319,8 +323,14 @@ ngx_http_init_connection(ngx_connection_ c->write->handler = ngx_http_empty_handler; #if (NGX_HTTP_V2) - if (hc->addr_conf->http2) { - rev->handler = ngx_http_v2_init; + { + ngx_http_v2_srv_conf_t *h2scf; + + h2scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v2_module); + + if (h2scf->enable || hc->addr_conf->http2) { + rev->handler = ngx_http_try_v2; + } } #endif @@ -638,6 +648,113 @@ ngx_http_alloc_request(ngx_connection_t } +#if (NGX_HTTP_V2) + +static void +ngx_http_try_v2(ngx_event_t *rev) +{ + u_char *p, buf[NGX_PROXY_PROTOCOL_MAX_HEADER + 1]; + size_t size; + ssize_t n; + ngx_err_t err; + ngx_connection_t *c; + ngx_http_connection_t *hc; + ngx_http_core_srv_conf_t *cscf; + + c = rev->data; + hc = c->data; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, rev->log, 0, "http check v2"); + + if (rev->timedout) { + ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, "client timed out"); + ngx_http_close_connection(c); + return; + } + + if (c->close) { + ngx_http_close_connection(c); + return; + } + + size = hc->proxy_protocol ? sizeof(buf) : 1; + + n = recv(c->fd, (char *) buf, size, MSG_PEEK); + + err = ngx_socket_errno; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, rev->log, 0, "http recv(): %z", n); + + if (n == -1) { + if (err == NGX_EAGAIN) { + rev->ready = 0; + + if (!rev->timer_set) { + cscf = ngx_http_get_module_srv_conf(hc->conf_ctx, + ngx_http_core_module); + ngx_add_timer(rev, cscf->client_header_timeout); + ngx_reusable_connection(c, 1); + } + + if (ngx_handle_read_event(rev, 0) != NGX_OK) { + ngx_http_close_connection(c); + } + + return; + } + + ngx_connection_error(c, err, "recv() failed"); + ngx_http_close_connection(c); + + return; + } + + if (hc->proxy_protocol) { + hc->proxy_protocol = 0; + + p = ngx_proxy_protocol_read(c, buf, buf + n); + + if (p == NULL) { + ngx_http_close_connection(c); + return; + } + + size = p - buf; + + if (c->recv(c, buf, size) != (ssize_t) size) { + ngx_http_close_connection(c); + return; + } + + if (n == (ssize_t) size) { + ngx_post_event(rev, &ngx_posted_events); + return; + } + + n = 1; + buf[0] = *p; + } + + if (n == 1) { + if (buf[0] == 'P') { + /* looks like HTTP/2 preface */ + ngx_http_v2_init(rev); + return; + } + + rev->handler = ngx_http_wait_request_handler; + ngx_http_wait_request_handler(rev); + + return; + } + + ngx_log_error(NGX_LOG_INFO, c->log, 0, "client closed connection"); + ngx_http_close_connection(c); +} + +#endif + + #if (NGX_HTTP_SSL) static void @@ -808,13 +925,16 @@ ngx_http_ssl_handshake_handler(ngx_conne #if (NGX_HTTP_V2 \ && defined TLSEXT_TYPE_application_layer_protocol_negotiation) { - unsigned int len; - const unsigned char *data; - ngx_http_connection_t *hc; + unsigned int len; + const unsigned char *data; + ngx_http_connection_t *hc; + ngx_http_v2_srv_conf_t *h2scf; hc = c->data; - if (hc->addr_conf->http2) { + h2scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v2_module); + + if (h2scf->enable || hc->addr_conf->http2) { SSL_get0_alpn_selected(c->ssl->connection, &data, &len); diff --git a/src/http/v2/ngx_http_v2.h b/src/http/v2/ngx_http_v2.h --- a/src/http/v2/ngx_http_v2.h +++ b/src/http/v2/ngx_http_v2.h @@ -64,6 +64,16 @@ typedef u_char *(*ngx_http_v2_handler_pt typedef struct { + ngx_flag_t enable; + size_t pool_size; + ngx_uint_t concurrent_streams; + ngx_uint_t concurrent_pushes; + size_t preread_size; + ngx_uint_t streams_index_mask; +} ngx_http_v2_srv_conf_t; + + +typedef struct { ngx_str_t name; ngx_str_t value; } ngx_http_v2_header_t; @@ -413,4 +423,7 @@ u_char *ngx_http_v2_string_encode(u_char u_char *tmp, ngx_uint_t lower); +extern ngx_module_t ngx_http_v2_module; + + #endif /* _NGX_HTTP_V2_H_INCLUDED_ */ diff --git a/src/http/v2/ngx_http_v2_module.c b/src/http/v2/ngx_http_v2_module.c --- a/src/http/v2/ngx_http_v2_module.c +++ b/src/http/v2/ngx_http_v2_module.c @@ -75,6 +75,13 @@ static ngx_conf_post_t ngx_http_v2_chun static ngx_command_t ngx_http_v2_commands[] = { + { ngx_string("http2"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v2_srv_conf_t, enable), + NULL }, + { ngx_string("http2_recv_buffer_size"), NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, ngx_conf_set_size_slot, @@ -314,6 +321,8 @@ ngx_http_v2_create_srv_conf(ngx_conf_t * return NULL; } + h2scf->enable = NGX_CONF_UNSET; + h2scf->pool_size = NGX_CONF_UNSET_SIZE; h2scf->concurrent_streams = NGX_CONF_UNSET_UINT; @@ -333,6 +342,8 @@ ngx_http_v2_merge_srv_conf(ngx_conf_t *c ngx_http_v2_srv_conf_t *prev = parent; ngx_http_v2_srv_conf_t *conf = child; + ngx_conf_merge_value(conf->enable, prev->enable, 0); + ngx_conf_merge_size_value(conf->pool_size, prev->pool_size, 4096); ngx_conf_merge_uint_value(conf->concurrent_streams, diff --git a/src/http/v2/ngx_http_v2_module.h b/src/http/v2/ngx_http_v2_module.h --- a/src/http/v2/ngx_http_v2_module.h +++ b/src/http/v2/ngx_http_v2_module.h @@ -21,15 +21,6 @@ typedef struct { typedef struct { - size_t pool_size; - ngx_uint_t concurrent_streams; - ngx_uint_t concurrent_pushes; - size_t preread_size; - ngx_uint_t streams_index_mask; -} ngx_http_v2_srv_conf_t; - - -typedef struct { size_t chunk_size; ngx_flag_t push_preload; @@ -39,7 +30,4 @@ typedef struct { } ngx_http_v2_loc_conf_t; -extern ngx_module_t ngx_http_v2_module; - - #endif /* _NGX_HTTP_V2_MODULE_H_INCLUDED_ */ From pluknet at nginx.com Thu Jan 26 16:00:53 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Thu, 26 Jan 2023 20:00:53 +0400 Subject: [PATCH] Added warning about redefinition of listen socket protocol options In-Reply-To: References: <5450B60D-DD89-4861-BA4D-6B57F085F7FD@nginx.com> Message-ID: > On 26 Jan 2023, at 01:29, Maxim Dounin wrote: > > Hello! > > On Mon, Jan 23, 2023 at 02:21:33PM +0400, Sergey Kandaurov wrote: > >>> On 31 Dec 2022, at 18:35, Maxim Dounin wrote: >>> >>> # HG changeset patch >>> # User Maxim Dounin >>> # Date 1672497248 -10800 >>> # Sat Dec 31 17:34:08 2022 +0300 >>> # Node ID c215d5cf25732ece1819cf1cd48ebb480bb642c7 >>> # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 >>> Added warning about redefinition of listen socket protocol options. >>> >>> The "listen" directive in the http module can be used multiple times >>> in different server blocks. Originally, it was supposed to be specified >>> once with various socket options, and without any parameters in virtual >>> server blocks. For example: >>> >>> server { listen 80 backlog=1024; server_name foo; ... } >>> server { listen 80; server_name bar; ... } >>> server { listen 80; server_name bazz; ... } >>> >>> The address part of the syntax ("address[:port]" / "port" / "unix:path") >>> uniquely identifies the listening socket, and therefore is enough for >>> name-based virtual servers (to let nginx know that the virtual server >>> accepts requests on the listening socket in question). >>> >>> To ensure that listening options do not conflict between virtual servers, >>> they were allowed only once. For example, the following configuration >>> will be rejected ("duplicate listen options for 0.0.0.0:80 in ..."): >>> >>> server { listen 80 backlog=1024; server_name foo; ... } >>> server { listen 80 backlog=512; server_name bar; ... } >>> >>> At some point it was, however, noticed, that it is sometimes convenient >>> to repeat some options for clarity. In nginx 0.8.51 the "ssl" parameter >>> was allowed to be specified multiple times, e.g.: >>> >>> server { listen 443 ssl backlog=1024; server_name foo; ... } >>> server { listen 443 ssl; server_name bar; ... } >>> server { listen 443 ssl; server_name bazz; ... } >>> >>> This approach makes configuration more readable, since SSL sockets are >>> immediately visible in the configuration. If this is not needed, just the >>> address can still be used. >>> >>> Later, additional protocol-specific options similar to "ssl" were >>> introduced, notably "http2" and "proxy_protocol". With these options, >>> one can write: >>> >>> server { listen 443 ssl backlog=1024; server_name foo; ... } >>> server { listen 443 http2; server_name bar; ... } >>> server { listen 443 proxy_protocol; server_name bazz; ... } >>> >>> The resulting socket will use ssl, http2, and proxy_protocol, but this >>> is not really obvious from the configuration. >>> >>> To ensure such misleading configurations are not allowed, nginx now >>> warns as long as the "listen" directive is used with options different >> >> nitpicking: >> >> "ensure .. are not allowed" and "warns" don't seem to be equally strong. >> As such, I'd rewrite to something like: >> >> To emphasize such misleading configurations are discouraged, nginx now > > Thanks, changed. > >>> from the options previously used if these are potentially confusing. >> >> Not really confident what "these" refers to. >> >> s/these/they/ ? > > I don't think that there is much difference between "these" and > "they" when it comes to what they refer to. Either way, "if this > is potentially confusing" is probably better and unambiguously > refers to the fact that the "listen" directive is used with the > different options. Changed. Sounds good as well, tnx. > >>> In particular, the following configurations are allowed: >>> >>> server { listen 8401 ssl backlog=1024; server_name foo; } >>> server { listen 8401 ssl; server_name bar; } >>> server { listen 8401 ssl; server_name bazz; } >>> >>> server { listen 8402 ssl http2 backlog=1024; server_name foo; } >>> server { listen 8402 ssl; server_name bar; } >>> server { listen 8402 ssl; server_name bazz; } >>> >>> server { listen 8403 ssl; server_name bar; } >>> server { listen 8403 ssl; server_name bazz; } >>> server { listen 8403 ssl http2; server_name foo; } >>> >>> server { listen 8404 ssl http2 backlog=1024; server_name foo; } >>> server { listen 8404 http2; server_name bar; } >>> server { listen 8404 http2; server_name bazz; } >>> >>> server { listen 8405 ssl http2 backlog=1024; server_name foo; } >>> server { listen 8405 ssl http2; server_name bar; } >>> server { listen 8405 ssl http2; server_name bazz; } >>> >>> server { listen 8406 ssl; server_name foo; } >>> server { listen 8406; server_name bar; } >>> server { listen 8406; server_name bazz; } >>> >>> And the following configurations will generate warnings: >>> >>> server { listen 8501 ssl http2 backlog=1024; server_name foo; } >>> server { listen 8501 http2; server_name bar; } >>> server { listen 8501 ssl; server_name bazz; } >>> >>> server { listen 8502 backlog=1024; server_name foo; } >>> server { listen 8502 ssl; server_name bar; } >>> >>> server { listen 8503 ssl; server_name foo; } >>> server { listen 8503 http2; server_name bar; } >>> >>> server { listen 8504 ssl; server_name foo; } >>> server { listen 8504 http2; server_name bar; } >>> server { listen 8504 proxy_protocol; server_name bazz; } >>> >>> server { listen 8505 ssl http2 proxy_protocol; server_name foo; } >>> server { listen 8505 ssl http2; server_name bar; } >>> server { listen 8505 ssl; server_name bazz; } >>> >>> server { listen 8506 ssl http2; server_name foo; } >>> server { listen 8506 ssl; server_name bar; } >>> server { listen 8506; server_name bazz; } >>> >>> server { listen 8507 ssl; server_name bar; } >>> server { listen 8507; server_name bazz; } >>> server { listen 8507 ssl http2; server_name foo; } >>> >>> server { listen 8508 ssl; server_name bar; } >>> server { listen 8508; server_name bazz; } >>> server { listen 8508 ssl backlog=1024; server_name foo; } >>> >>> server { listen 8509; server_name bazz; } >>> server { listen 8509 ssl; server_name bar; } >>> server { listen 8509 ssl backlog=1024; server_name foo; } >>> >> >> 15 examples of dos and don'ts looks slightly excessive. >> The accurate description (such as provided by you below) allows >> to reduce most of them to e.g. four common invalid configurations: >> >> A lesser option set with socket option: >> >> server { listen 8443 backlog=1024; server_name foo; } >> server { listen 8443 http2; server_name bar; } >> >> The main option set is repeated at least twice: >> >> server { listen 127.0.0.1:8443; server_name foo; } >> server { listen 127.0.0.1:8443 ssl; server_name bar; } >> server { listen 127.0.0.1:8443 ssl; server_name baz; } >> >> Option sets partially overlap: >> >> server { listen 127.0.0.1:8443 ssl; server_name foo; } >> server { listen 127.0.0.1:8443 http2; server_name bar; } >> >> More than two option sets: >> >> server { listen 127.0.0.1:8443 http2 ssl; server_name foo; } >> server { listen 127.0.0.1:8443 http2; server_name bar; } >> server { listen 127.0.0.1:8443 ssl; server_name baz; } > > While your approach might be better from documentation point > of view, the way it is currently described in the commit log is > how it was designed: from the examples of valid and invalid > configurations. > > My current working tests contain 18 valid and 22 invalid > configurations, derived from the ones provided in the commit log > with additional shuffling. But since these are derived I've > decided to avoid providing all of them in the commit log. > >>> The basic idea is that at most two sets of protocol options are allowed: >>> the main one (with socket options, if any), and a shorter one, with options >>> being a subset of the main options, repeated for clarity. As long as the >>> shorter set of protocol options is used, all listen directives except the >>> main one should use it. >> >> I'd move this paragraph somewhere before examples, as this is the most >> specific description of things actually changed. > > This paragraph summarizes changes made to address the issue > described, and I don't think moving it will improve things. The above looks sane now, thanks for the explanation. I won't insist on further adjustments. > >> BTW, while reviewing I caught sort of a bug. >> As I understand the above explanation, if there are both full and short >> sets present, then at most one listen directive can have the full set, >> while shorter sets can be repeated. If so, then with the proposed patch >> the next configuration is expectedly invalid: >> >> server { listen 127.0.0.1:8443 ssl; server_name foo; } >> server { listen 127.0.0.1:8443 ssl; server_name bar; } >> server { listen 127.0.0.1:8443; server_name baz; } >> >> This is expected since first two servers with same options are >> interpreted as a short form (with full form seen potentially later on), >> but 3rd server has lesser options, which is caught by this check: >> (addr[i].protocols_set && protocols != addr[i].protocols) >> Which is interpreted as: >> "this server has a lesser set that doesn't match a a shorter set". >> >> Now, if 3rd server is moved first, configuration starts to pass: >> >> server { listen 127.0.0.1:8443; server_name baz; } >> server { listen 127.0.0.1:8443 ssl; server_name foo; } >> server { listen 127.0.0.1:8443 ssl; server_name bar; } >> >> This is because after (now) 2nd server, it is parsed as: >> 1st server has a short form, and 2nd server has a full form. >> Then 3rd server goes to "the same options" case. This also >> overwrites the remembered shorter set in addr[i].protocols. >> >> I guess an additional check should be added to address this. >> It is similar to the "options removed" case and ensures that >> the repeated options set actually matches a shorter set: >> >> diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c >> --- a/src/http/ngx_http.c >> +++ b/src/http/ngx_http.c >> @@ -1345,7 +1345,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, n >> >> /* the same options */ >> >> - if (lsopt->set && addr[i].protocols_changed) { >> + if ((lsopt->set && addr[i].protocols_changed) >> + || (addr[i].protocols_set && protocols != addr[i].protocols)) >> + { >> ngx_conf_log_error(NGX_LOG_WARN, cf, 0, >> "protocol options redefined for %V", >> &addr[i].opt.addr_text); > > Thanks for catching this, applied. > > Updated patch: > > # HG changeset patch > # User Maxim Dounin > # Date 1674624978 -10800 > # Wed Jan 25 08:36:18 2023 +0300 > # Node ID 1619d1563c3231d4283937610ce97aa1e3824fb8 > # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db > Added warning about redefinition of listen socket protocol options. > > The "listen" directive in the http module can be used multiple times > in different server blocks. Originally, it was supposed to be specified > once with various socket options, and without any parameters in virtual > server blocks. For example: > > server { listen 80 backlog=1024; server_name foo; ... } > server { listen 80; server_name bar; ... } > server { listen 80; server_name bazz; ... } > > The address part of the syntax ("address[:port]" / "port" / "unix:path") > uniquely identifies the listening socket, and therefore is enough for > name-based virtual servers (to let nginx know that the virtual server > accepts requests on the listening socket in question). > > To ensure that listening options do not conflict between virtual servers, > they were allowed only once. For example, the following configuration > will be rejected ("duplicate listen options for 0.0.0.0:80 in ..."): > > server { listen 80 backlog=1024; server_name foo; ... } > server { listen 80 backlog=512; server_name bar; ... } > > At some point it was, however, noticed, that it is sometimes convenient > to repeat some options for clarity. In nginx 0.8.51 the "ssl" parameter > was allowed to be specified multiple times, e.g.: > > server { listen 443 ssl backlog=1024; server_name foo; ... } > server { listen 443 ssl; server_name bar; ... } > server { listen 443 ssl; server_name bazz; ... } > > This approach makes configuration more readable, since SSL sockets are > immediately visible in the configuration. If this is not needed, just the > address can still be used. > > Later, additional protocol-specific options similar to "ssl" were > introduced, notably "http2" and "proxy_protocol". With these options, > one can write: > > server { listen 443 ssl backlog=1024; server_name foo; ... } > server { listen 443 http2; server_name bar; ... } > server { listen 443 proxy_protocol; server_name bazz; ... } > > The resulting socket will use ssl, http2, and proxy_protocol, but this > is not really obvious from the configuration. > > To emphasize such misleading configurations are discouraged, nginx now > warns as long as the "listen" directive is used with options different > from the options previously used if this is potentially confusing. > > In particular, the following configurations are allowed: > > server { listen 8401 ssl backlog=1024; server_name foo; } > server { listen 8401 ssl; server_name bar; } > server { listen 8401 ssl; server_name bazz; } > > server { listen 8402 ssl http2 backlog=1024; server_name foo; } > server { listen 8402 ssl; server_name bar; } > server { listen 8402 ssl; server_name bazz; } > > server { listen 8403 ssl; server_name bar; } > server { listen 8403 ssl; server_name bazz; } > server { listen 8403 ssl http2; server_name foo; } > > server { listen 8404 ssl http2 backlog=1024; server_name foo; } > server { listen 8404 http2; server_name bar; } > server { listen 8404 http2; server_name bazz; } > > server { listen 8405 ssl http2 backlog=1024; server_name foo; } > server { listen 8405 ssl http2; server_name bar; } > server { listen 8405 ssl http2; server_name bazz; } > > server { listen 8406 ssl; server_name foo; } > server { listen 8406; server_name bar; } > server { listen 8406; server_name bazz; } > > And the following configurations will generate warnings: > > server { listen 8501 ssl http2 backlog=1024; server_name foo; } > server { listen 8501 http2; server_name bar; } > server { listen 8501 ssl; server_name bazz; } > > server { listen 8502 backlog=1024; server_name foo; } > server { listen 8502 ssl; server_name bar; } > > server { listen 8503 ssl; server_name foo; } > server { listen 8503 http2; server_name bar; } > > server { listen 8504 ssl; server_name foo; } > server { listen 8504 http2; server_name bar; } > server { listen 8504 proxy_protocol; server_name bazz; } > > server { listen 8505 ssl http2 proxy_protocol; server_name foo; } > server { listen 8505 ssl http2; server_name bar; } > server { listen 8505 ssl; server_name bazz; } > > server { listen 8506 ssl http2; server_name foo; } > server { listen 8506 ssl; server_name bar; } > server { listen 8506; server_name bazz; } > > server { listen 8507 ssl; server_name bar; } > server { listen 8507; server_name bazz; } > server { listen 8507 ssl http2; server_name foo; } > > server { listen 8508 ssl; server_name bar; } > server { listen 8508; server_name bazz; } > server { listen 8508 ssl backlog=1024; server_name foo; } > > server { listen 8509; server_name bazz; } > server { listen 8509 ssl; server_name bar; } > server { listen 8509 ssl backlog=1024; server_name foo; } > > The basic idea is that at most two sets of protocol options are allowed: > the main one (with socket options, if any), and a shorter one, with options > being a subset of the main options, repeated for clarity. As long as the > shorter set of protocol options is used, all listen directives except the > main one should use it. > > diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c > --- a/src/http/ngx_http.c > +++ b/src/http/ngx_http.c > @@ -1228,7 +1228,8 @@ static ngx_int_t > ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, > ngx_http_conf_port_t *port, ngx_http_listen_opt_t *lsopt) > { > - ngx_uint_t i, default_server, proxy_protocol; > + ngx_uint_t i, default_server, proxy_protocol, > + protocols, protocols_prev; > ngx_http_conf_addr_t *addr; > #if (NGX_HTTP_SSL) > ngx_uint_t ssl; > @@ -1264,12 +1265,18 @@ ngx_http_add_addresses(ngx_conf_t *cf, n > default_server = addr[i].opt.default_server; > > proxy_protocol = lsopt->proxy_protocol || addr[i].opt.proxy_protocol; > + protocols = lsopt->proxy_protocol; > + protocols_prev = addr[i].opt.proxy_protocol; > > #if (NGX_HTTP_SSL) > ssl = lsopt->ssl || addr[i].opt.ssl; > + protocols |= lsopt->ssl << 1; > + protocols_prev |= addr[i].opt.ssl << 1; > #endif > #if (NGX_HTTP_V2) > http2 = lsopt->http2 || addr[i].opt.http2; > + protocols |= lsopt->http2 << 2; > + protocols_prev |= addr[i].opt.http2 << 2; > #endif > > if (lsopt->set) { > @@ -1299,6 +1306,57 @@ ngx_http_add_addresses(ngx_conf_t *cf, n > addr[i].default_server = cscf; > } > > + /* check for conflicting protocol options */ > + > + if ((protocols | protocols_prev) != protocols_prev) { > + > + /* options added */ > + > + if ((addr[i].opt.set && !lsopt->set) > + || addr[i].protocols_changed > + || (protocols | protocols_prev) != protocols) > + { > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > + "protocol options redefined for %V", > + &addr[i].opt.addr_text); > + } > + > + addr[i].protocols = protocols_prev; > + addr[i].protocols_set = 1; > + addr[i].protocols_changed = 1; > + > + } else if ((protocols_prev | protocols) != protocols) { > + > + /* options removed */ > + > + if (lsopt->set > + || (addr[i].protocols_set && protocols != addr[i].protocols)) > + { > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > + "protocol options redefined for %V", > + &addr[i].opt.addr_text); > + } > + > + addr[i].protocols = protocols; > + addr[i].protocols_set = 1; > + addr[i].protocols_changed = 1; > + > + } else { > + > + /* the same options */ > + > + if ((lsopt->set && addr[i].protocols_changed) > + || (addr[i].protocols_set && protocols != addr[i].protocols)) > + { > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > + "protocol options redefined for %V", > + &addr[i].opt.addr_text); > + } > + > + addr[i].protocols = protocols; > + addr[i].protocols_set = 1; > + } > + > addr[i].opt.default_server = default_server; > addr[i].opt.proxy_protocol = proxy_protocol; > #if (NGX_HTTP_SSL) > @@ -1355,6 +1413,9 @@ ngx_http_add_address(ngx_conf_t *cf, ngx > } > > addr->opt = *lsopt; > + addr->protocols = 0; > + addr->protocols_set = 0; > + addr->protocols_changed = 0; > addr->hash.buckets = NULL; > addr->hash.size = 0; > addr->wc_head = NULL; > diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h > --- a/src/http/ngx_http_core_module.h > +++ b/src/http/ngx_http_core_module.h > @@ -274,6 +274,10 @@ typedef struct { > typedef struct { > ngx_http_listen_opt_t opt; > > + unsigned protocols:3; > + unsigned protocols_set:1; > + unsigned protocols_changed:1; > + > ngx_hash_t hash; > ngx_hash_wildcard_t *wc_head; > ngx_hash_wildcard_t *wc_tail; > Looks good now, thanks. -- Sergey Kandaurov From mdounin at mdounin.ru Fri Jan 27 04:01:23 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Fri, 27 Jan 2023 07:01:23 +0300 Subject: [PATCH] HTTP: trigger lingering close when keepalive connection will be closed In-Reply-To: References: <699749D2-D96F-47D8-85DA-74E7F97682FF@gmail.com> <8D59E87C-2DC3-43AB-9067-019CA9ACE5C4@gmail.com> Message-ID: Hello! On Wed, Jan 25, 2023 at 12:49:10PM +0800, Miao Wang wrote: > > 2023年1月25日 10:17，Maxim Dounin 写道： > > > > On Mon, Jan 23, 2023 at 07:01:16PM +0800, Miao Wang wrote: > > > >>> 2023年1月23日 12:05，Maxim Dounin 写道： > >>> > >>> On Wed, Jan 18, 2023 at 11:28:52PM +0800, Miao Wang wrote: > >>> > >>>> # HG changeset patch > >>>> # User Miao Wang > >>>> # Date 1674055068 -28800 > >>>> # Wed Jan 18 23:17:48 2023 +0800 > >>>> # Node ID 73aa64bd29f3dec9e43e97560d6b5a07cdf40063 > >>>> # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > >>>> HTTP: trigger lingering close when keepalive connection will be closed > >>>> > >>>> When finalizing a request, if the request is not keepalive but > >>>> its connection has served more than one request, then the connection > >>>> has been a keepalive connection previously and this connection will > >>>> be closed after this response. In this condition, it is likely that > >>>> there are pipelined requests following this request, which we should > >>>> ignore. As a result, lingering close is necessary in this case. > >>>> > >>>> Without this patch, nginx (with its default configuration) will send > >>>> out TCP RST when there are more pipelined requests. The symptom is > >>>> obvious when nginx is serving a debian repository and apt is > >>>> downloading massive of packages. See [1]. It becomes more obvious > >>>> when `keepalive_requests` is lower or nginx is under a relative > >>>> higher load, and it disappears when specifying > >>>> `lingering_close always`. > >>>> > >>>> [1]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 > >>>> > >>>> diff -r 07b0bee87f32 -r 73aa64bd29f3 src/http/ngx_http_request.c > >>>> --- a/src/http/ngx_http_request.c Wed Dec 21 14:53:27 2022 +0300 > >>>> +++ b/src/http/ngx_http_request.c Wed Jan 18 23:17:48 2023 +0800 > >>>> @@ -2749,6 +2749,10 @@ > >>>> return; > >>>> } > >>>> > >>>> + if (!r->keepalive && r->connection->requests > 1) { > >>>> + r->lingering_close = 1; > >>>> + } > >>>> + > >>>> if (clcf->lingering_close == NGX_HTTP_LINGERING_ALWAYS > >>>> || (clcf->lingering_close == NGX_HTTP_LINGERING_ON > >>>> && (r->lingering_close > >>> > >>> Thanks for the patch and the link to the Debian bug report. > >>> > >>> Lingering close implies noticeable additional resource usage: even > >>> if nothing happens on the connection, it will be kept open for > >>> lingering_timeout, which is 5 seconds by default. Given that > >>> pipelining is not used by most of the clients, forcing lingering > >>> close for all clients which are using keepalive does not look like > >>> a good solution to me. > >>> > >>> In general, nginx tries hard to determine if any additional data > >>> are expected on the connection, and uses lingering close if there > >>> is a good chance there will be some, but avoids lingering close by > >>> default if additional data are unlikely. If this logic does not > >>> work for some reason, lingering close can be explicitly requested > >>> with "lingering_close always;". > >> > >> That's true since the symptom I described can be worked around with > >> that option. > >> > >>> > >>> In particular, note the "r->header_in->pos < r->header_in->last" > >>> and "r->connection->read->ready" checks - these are expected to > >>> catch connections with additional pipelined requests (see revision > >>> 3981:77604e9a1ed8). And from the log provided in the report it > >>> looks like it works most of the time - there are more than 6k HTTP > >>> requests, and 60+ connections. But sometimes it fails - there are > >>> two RST errors logged (and one "Undetermined Error", which looks > >>> like a bug in apt, but might be related). > >>> > >>> It looks like when apt is downloading many resources, it does not > >>> send all the requests at once (or in batches), but instead tries > >>> to maintain a constant "depth", a number of pipelined requests in > >>> flight. This essentially means that after reading of a response > >>> it sends an additional request. > >> > >> That's right. From a traffic dump, I can see apt first sends one > >> request, and after receiving the response, it will send out 10 > >> more requests, and maintain a depth of 10, since by default > >> Acquire::http::Pipeline-Depth is 10. > >> > >>> > >>> I see at least two possible cases which can result in nginx not > >>> using lingering close with such a load: > >>> > >>> 1. If a response where keepalive_requests is reached happens to > >>> be the last request in the r->header_in buffer (so the > >>> "r->header_in->pos < r->header_in->last" won't be effective), and > >>> there is a chance that nginx wasn't yet got an event from kernel > >>> about additional data (and therefore "r->connection->read->ready" > >>> will not be set). As such, nginx won't use lingering close, and > >>> might close connection with unread data in the socket buffer, > >>> resulting in RST. > >>> > >>> 2. Similarly, if nginx happens to be faster than apt, and socket > >>> buffers are large enough, it might sent all the responses, > >>> including the last one with "Connection: close", and close the > >>> connection (since there are no pending pipelined requests at the > >>> moment) even before an additional request is sent by apt. When > >>> later apt will send an additional request after reading some of > >>> the responses, it will send the request to already closed > >>> connection, again resulting in RST. > >> > >> Actually, comparing the debug log and the pcap, nginx calls > >> close() after writing the last response. However, at that time, > >> that response is not fully transmitted to the client and there > >> seems to be more requests not processed in the kernel buffer. > >> Thus close() triggers an immediate RST. > > > > Thanks for the details. This looks more like the first case, and > > probably can be addressed by improving likelihood of detecting the > > read event. > > > > Could you please test if the patch below fixes the particular > > issue you are seeing? It is somewhat unrelated, but it might be > > a good enough solution (and is more or less equivalent to > > checking r->pipeline). > > > >>> It would be interesting to see more details, such as tcpdump > >>> and/or nginx debug logs, to find out what actually goes on here. > >> > >> The tcpdump and debug logs are too large to send in this mail list. > >> I wonder if I can directly email it to you. > > > > Feel free to, my email should accept up to 100M messages. > > Alternatively, a good solution might be to make the files > > available for download and post a link here. > > > >>> Overall, given how apt uses pipelining, I tend to think that at > >>> least (2) is unavoidable and can happen with certain sizes of the > >>> responses. > >>> > >>> A good enough solution might be check for r->pipeline, which is > >>> set by nginx as long as it reads a pipelined request. It might > >>> not be enough though, since r->pipeline is only set for requests > >>> seen by nginx as pipelined, which might not be true for the last > >>> request. > >>> > >>> A more complete solution might be to introduce something like > >>> c->pipeline flag and use lingering close if any pipelined requests > >>> were seen on the connection. > > > > The following patch reworks handling of pipelined requests by > > postponing them to the next event loop iteration. It is expected > > make it more likely for nginx to know there are any additional > > unread data in the socket buffer (and right now is mostly > > equivalent to checking r->pipeline, since c->read->ready is always > > set for pipelined requests): > > > > # HG changeset patch > > # User Maxim Dounin > > # Date 1674610218 -10800 > > # Wed Jan 25 04:30:18 2023 +0300 > > # Node ID 8cfd22c325a3db370b9e45aa6f897ff7bc8222f3 > > # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db > > Reworked pipelined requests to use posted next events. > > > > This is expected to improve handling of pipelined requests in a number > > of ways, including: > > > > 1) It will make a room for additional requests from other clients, > > reducing worker monopolization by a single client. > > > > 2) The c->read->ready flag will be set, so nginx will either read the > > additional data, or will use lingering close. This is expected to help > > with clients using pipelining with some constant depth, such as apt[1][2]. > > > > The ngx_event_move_posted_next() was modified to make it possible to > > post read events on connections with kqueue. Previously, it used to > > set ev->available to -1, potentially overwriting a valid positive value > > provided by kqueue, so ngx_unix_recv() and ngx_readv_chain() will stop > > reading from the socket before reading all the data available. > > > > Note that currently ngx_event_move_posted_next() will always set > > the ev->ready flag. While this is expected behaviour for the ev->available > > use case (where ev->ready is explicitly cleared), this is not needed for > > pipelining. For pipelining, this will result in extra unneeded read() > > syscall after processing of all pipelined requests, and there might be > > a room for improvement here. > > > > [1] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 > > [2] https://mailman.nginx.org/pipermail/nginx-devel/2023-January/ZA2SP5SJU55LHEBCJMFDB2AZVELRLTHI.html > > > > diff --git a/src/event/ngx_event_posted.c b/src/event/ngx_event_posted.c > > --- a/src/event/ngx_event_posted.c > > +++ b/src/event/ngx_event_posted.c > > @@ -51,8 +51,10 @@ ngx_event_move_posted_next(ngx_cycle_t * > > ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, > > "posted next event %p", ev); > > > > - ev->ready = 1; > > - ev->available = -1; > > + if (!ev->ready) { > > + ev->ready = 1; > > + ev->available = -1; > > + } > > } > > > > ngx_queue_add(&ngx_posted_events, &ngx_posted_next_events); > > diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c > > --- a/src/http/ngx_http_request.c > > +++ b/src/http/ngx_http_request.c > > @@ -3129,7 +3129,7 @@ ngx_http_set_keepalive(ngx_http_request_ > > } > > > > rev->handler = ngx_http_process_request_line; > > - ngx_post_event(rev, &ngx_posted_events); > > + ngx_post_event(rev, &ngx_posted_next_events); > > return; > > } > > > > I can confirm that the symptom disappears after applying this patch Thanks for testing, and thanks for the logs and traffic dumps provided. >From the logs you've provided it looks like the patch should help with 1 out of 5 errors: the last request in the connection with this error was pipelined, and therefore with the patch it is expected to use lingering close. In 4 other cases the last requests are handled without pipelining: once the request arrives, ngx_http_keepalive_handler() is called, and nginx reads the request and responds to it, closing the connection. The next request arrives at some time later, and the OS responds with RST. E.g., the request to bzip2_1.0.8-4_amd64.deb (which corresponds to the second error as seen by apt, while downloading libpython3.9-stdlib_3.9.2-1_amd64.deb; connection *26, with client port 40110) arrives at 14:08:36.495858, nginx responds to it, as seen in nginx logs, and then closes the connection. The next request arrives at 14:08:36.496977 and immediately responded with RST. The patch does not change handling of these 4 cases, except may be some mostly unrelated event handling changes, so the timing might be slightly different. As such, I would expect the errors to be still reproducible with the patch, but probably somewhat less often. Could you please re-check to see if the symptoms still happen with the patch, at least occasionally? Overall, after looking into logs and tcpdump you've provided I tend to think that the only working fix would be to introduce c->pipeline flag, and force lingering close if there were any pipelined requests on the connection. Below is the patch which implements this approach. Review and testing appreciated. It can be used either separately or with the previously provided patch to use posted next events. # HG changeset patch # User Maxim Dounin # Date 1674790916 -10800 # Fri Jan 27 06:41:56 2023 +0300 # Node ID 784d0fa0b5a0796561642a5a64dc4e9e07592852 # Parent 4eb1383f6432b034630e6de18739b817f6565c8c Lingering close for connections with pipelined requests. This is expected to help with clients using pipelining with some constant depth, such as apt[1][2]. When downloading many resources, apt uses pipelining with some constant depth, a number of requests in flight[1][2]. This essentially means that after receiving a response it sends an additional request to the server, and this can result in requests arriving to the server at any time. Further, additional requests are sent one-by-one, and can be easily seen as such (neither as pipelined, nor followed by pipelined requests). The only safe approach to close such connections (for example, when keepalive_requests is reached) is with lingering. To do so, now nginx monitors if pipelining was used on the connection, and if it was, closes the connection with lingering. [1] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 [2] https://mailman.nginx.org/pipermail/nginx-devel/2023-January/ZA2SP5SJU55LHEBCJMFDB2AZVELRLTHI.html diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h --- a/src/core/ngx_connection.h +++ b/src/core/ngx_connection.h @@ -172,6 +172,7 @@ struct ngx_connection_s { unsigned timedout:1; unsigned error:1; unsigned destroyed:1; + unsigned pipeline:1; unsigned idle:1; unsigned reusable:1; diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -2753,7 +2753,8 @@ ngx_http_finalize_connection(ngx_http_re || (clcf->lingering_close == NGX_HTTP_LINGERING_ON && (r->lingering_close || r->header_in->pos < r->header_in->last - || r->connection->read->ready))) + || r->connection->read->ready + || r->connection->pipeline))) { ngx_http_set_lingering_close(r->connection); return; @@ -3123,6 +3124,7 @@ ngx_http_set_keepalive(ngx_http_request_ c->sent = 0; c->destroyed = 0; + c->pipeline = 1; if (rev->timer_set) { ngx_del_timer(rev); -- Maxim Dounin http://mdounin.ru/ From shankerwangmiao at gmail.com Fri Jan 27 12:11:34 2023 From: shankerwangmiao at gmail.com (Miao Wang) Date: Fri, 27 Jan 2023 20:11:34 +0800 Subject: [PATCH] HTTP: trigger lingering close when keepalive connection will be closed In-Reply-To: References: <699749D2-D96F-47D8-85DA-74E7F97682FF@gmail.com> <8D59E87C-2DC3-43AB-9067-019CA9ACE5C4@gmail.com> Message-ID: <01045AAC-62AA-4DEB-84D0-9C65754FD639@gmail.com> Hi, > 2023年1月27日 12:01，Maxim Dounin 写道： > > Hello! > > On Wed, Jan 25, 2023 at 12:49:10PM +0800, Miao Wang wrote: > >>> 2023年1月25日 10:17，Maxim Dounin 写道： >>> >>> On Mon, Jan 23, 2023 at 07:01:16PM +0800, Miao Wang wrote: >>> >>>>> 2023年1月23日 12:05，Maxim Dounin 写道： >>>>> >>>>> On Wed, Jan 18, 2023 at 11:28:52PM +0800, Miao Wang wrote: >>>>> >>>>>> # HG changeset patch >>>>>> # User Miao Wang >>>>>> # Date 1674055068 -28800 >>>>>> # Wed Jan 18 23:17:48 2023 +0800 >>>>>> # Node ID 73aa64bd29f3dec9e43e97560d6b5a07cdf40063 >>>>>> # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 >>>>>> HTTP: trigger lingering close when keepalive connection will be closed >>>>>> >>>>>> When finalizing a request, if the request is not keepalive but >>>>>> its connection has served more than one request, then the connection >>>>>> has been a keepalive connection previously and this connection will >>>>>> be closed after this response. In this condition, it is likely that >>>>>> there are pipelined requests following this request, which we should >>>>>> ignore. As a result, lingering close is necessary in this case. >>>>>> >>>>>> Without this patch, nginx (with its default configuration) will send >>>>>> out TCP RST when there are more pipelined requests. The symptom is >>>>>> obvious when nginx is serving a debian repository and apt is >>>>>> downloading massive of packages. See [1]. It becomes more obvious >>>>>> when `keepalive_requests` is lower or nginx is under a relative >>>>>> higher load, and it disappears when specifying >>>>>> `lingering_close always`. >>>>>> >>>>>> [1]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 >>>>>> >>>>>> diff -r 07b0bee87f32 -r 73aa64bd29f3 src/http/ngx_http_request.c >>>>>> --- a/src/http/ngx_http_request.c Wed Dec 21 14:53:27 2022 +0300 >>>>>> +++ b/src/http/ngx_http_request.c Wed Jan 18 23:17:48 2023 +0800 >>>>>> @@ -2749,6 +2749,10 @@ >>>>>> return; >>>>>> } >>>>>> >>>>>> + if (!r->keepalive && r->connection->requests > 1) { >>>>>> + r->lingering_close = 1; >>>>>> + } >>>>>> + >>>>>> if (clcf->lingering_close == NGX_HTTP_LINGERING_ALWAYS >>>>>> || (clcf->lingering_close == NGX_HTTP_LINGERING_ON >>>>>> && (r->lingering_close >>>>> >>>>> Thanks for the patch and the link to the Debian bug report. >>>>> >>>>> Lingering close implies noticeable additional resource usage: even >>>>> if nothing happens on the connection, it will be kept open for >>>>> lingering_timeout, which is 5 seconds by default. Given that >>>>> pipelining is not used by most of the clients, forcing lingering >>>>> close for all clients which are using keepalive does not look like >>>>> a good solution to me. >>>>> >>>>> In general, nginx tries hard to determine if any additional data >>>>> are expected on the connection, and uses lingering close if there >>>>> is a good chance there will be some, but avoids lingering close by >>>>> default if additional data are unlikely. If this logic does not >>>>> work for some reason, lingering close can be explicitly requested >>>>> with "lingering_close always;". >>>> >>>> That's true since the symptom I described can be worked around with >>>> that option. >>>> >>>>> >>>>> In particular, note the "r->header_in->pos < r->header_in->last" >>>>> and "r->connection->read->ready" checks - these are expected to >>>>> catch connections with additional pipelined requests (see revision >>>>> 3981:77604e9a1ed8). And from the log provided in the report it >>>>> looks like it works most of the time - there are more than 6k HTTP >>>>> requests, and 60+ connections. But sometimes it fails - there are >>>>> two RST errors logged (and one "Undetermined Error", which looks >>>>> like a bug in apt, but might be related). >>>>> >>>>> It looks like when apt is downloading many resources, it does not >>>>> send all the requests at once (or in batches), but instead tries >>>>> to maintain a constant "depth", a number of pipelined requests in >>>>> flight. This essentially means that after reading of a response >>>>> it sends an additional request. >>>> >>>> That's right. From a traffic dump, I can see apt first sends one >>>> request, and after receiving the response, it will send out 10 >>>> more requests, and maintain a depth of 10, since by default >>>> Acquire::http::Pipeline-Depth is 10. >>>> >>>>> >>>>> I see at least two possible cases which can result in nginx not >>>>> using lingering close with such a load: >>>>> >>>>> 1. If a response where keepalive_requests is reached happens to >>>>> be the last request in the r->header_in buffer (so the >>>>> "r->header_in->pos < r->header_in->last" won't be effective), and >>>>> there is a chance that nginx wasn't yet got an event from kernel >>>>> about additional data (and therefore "r->connection->read->ready" >>>>> will not be set). As such, nginx won't use lingering close, and >>>>> might close connection with unread data in the socket buffer, >>>>> resulting in RST. >>>>> >>>>> 2. Similarly, if nginx happens to be faster than apt, and socket >>>>> buffers are large enough, it might sent all the responses, >>>>> including the last one with "Connection: close", and close the >>>>> connection (since there are no pending pipelined requests at the >>>>> moment) even before an additional request is sent by apt. When >>>>> later apt will send an additional request after reading some of >>>>> the responses, it will send the request to already closed >>>>> connection, again resulting in RST. >>>> >>>> Actually, comparing the debug log and the pcap, nginx calls >>>> close() after writing the last response. However, at that time, >>>> that response is not fully transmitted to the client and there >>>> seems to be more requests not processed in the kernel buffer. >>>> Thus close() triggers an immediate RST. >>> >>> Thanks for the details. This looks more like the first case, and >>> probably can be addressed by improving likelihood of detecting the >>> read event. >>> >>> Could you please test if the patch below fixes the particular >>> issue you are seeing? It is somewhat unrelated, but it might be >>> a good enough solution (and is more or less equivalent to >>> checking r->pipeline). >>> >>>>> It would be interesting to see more details, such as tcpdump >>>>> and/or nginx debug logs, to find out what actually goes on here. >>>> >>>> The tcpdump and debug logs are too large to send in this mail list. >>>> I wonder if I can directly email it to you. >>> >>> Feel free to, my email should accept up to 100M messages. >>> Alternatively, a good solution might be to make the files >>> available for download and post a link here. >>> >>>>> Overall, given how apt uses pipelining, I tend to think that at >>>>> least (2) is unavoidable and can happen with certain sizes of the >>>>> responses. >>>>> >>>>> A good enough solution might be check for r->pipeline, which is >>>>> set by nginx as long as it reads a pipelined request. It might >>>>> not be enough though, since r->pipeline is only set for requests >>>>> seen by nginx as pipelined, which might not be true for the last >>>>> request. >>>>> >>>>> A more complete solution might be to introduce something like >>>>> c->pipeline flag and use lingering close if any pipelined requests >>>>> were seen on the connection. >>> >>> The following patch reworks handling of pipelined requests by >>> postponing them to the next event loop iteration. It is expected >>> make it more likely for nginx to know there are any additional >>> unread data in the socket buffer (and right now is mostly >>> equivalent to checking r->pipeline, since c->read->ready is always >>> set for pipelined requests): >>> >>> # HG changeset patch >>> # User Maxim Dounin >>> # Date 1674610218 -10800 >>> # Wed Jan 25 04:30:18 2023 +0300 >>> # Node ID 8cfd22c325a3db370b9e45aa6f897ff7bc8222f3 >>> # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db >>> Reworked pipelined requests to use posted next events. >>> >>> This is expected to improve handling of pipelined requests in a number >>> of ways, including: >>> >>> 1) It will make a room for additional requests from other clients, >>> reducing worker monopolization by a single client. >>> >>> 2) The c->read->ready flag will be set, so nginx will either read the >>> additional data, or will use lingering close. This is expected to help >>> with clients using pipelining with some constant depth, such as apt[1][2]. >>> >>> The ngx_event_move_posted_next() was modified to make it possible to >>> post read events on connections with kqueue. Previously, it used to >>> set ev->available to -1, potentially overwriting a valid positive value >>> provided by kqueue, so ngx_unix_recv() and ngx_readv_chain() will stop >>> reading from the socket before reading all the data available. >>> >>> Note that currently ngx_event_move_posted_next() will always set >>> the ev->ready flag. While this is expected behaviour for the ev->available >>> use case (where ev->ready is explicitly cleared), this is not needed for >>> pipelining. For pipelining, this will result in extra unneeded read() >>> syscall after processing of all pipelined requests, and there might be >>> a room for improvement here. >>> >>> [1] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 >>> [2] https://mailman.nginx.org/pipermail/nginx-devel/2023-January/ZA2SP5SJU55LHEBCJMFDB2AZVELRLTHI.html >>> >>> diff --git a/src/event/ngx_event_posted.c b/src/event/ngx_event_posted.c >>> --- a/src/event/ngx_event_posted.c >>> +++ b/src/event/ngx_event_posted.c >>> @@ -51,8 +51,10 @@ ngx_event_move_posted_next(ngx_cycle_t * >>> ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, >>> "posted next event %p", ev); >>> >>> - ev->ready = 1; >>> - ev->available = -1; >>> + if (!ev->ready) { >>> + ev->ready = 1; >>> + ev->available = -1; >>> + } >>> } >>> >>> ngx_queue_add(&ngx_posted_events, &ngx_posted_next_events); >>> diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c >>> --- a/src/http/ngx_http_request.c >>> +++ b/src/http/ngx_http_request.c >>> @@ -3129,7 +3129,7 @@ ngx_http_set_keepalive(ngx_http_request_ >>> } >>> >>> rev->handler = ngx_http_process_request_line; >>> - ngx_post_event(rev, &ngx_posted_events); >>> + ngx_post_event(rev, &ngx_posted_next_events); >>> return; >>> } >>> >> >> I can confirm that the symptom disappears after applying this patch > > Thanks for testing, and thanks for the logs and traffic dumps > provided. > > From the logs you've provided it looks like the patch should help > with 1 out of 5 errors: the last request in the connection with > this error was pipelined, and therefore with the patch it is > expected to use lingering close. > > In 4 other cases the last requests are handled without pipelining: > once the request arrives, ngx_http_keepalive_handler() is called, > and nginx reads the request and responds to it, closing the > connection. The next request arrives at some time later, and the > OS responds with RST. > > E.g., the request to bzip2_1.0.8-4_amd64.deb (which corresponds to > the second error as seen by apt, while downloading > libpython3.9-stdlib_3.9.2-1_amd64.deb; connection *26, with client > port 40110) arrives at 14:08:36.495858, nginx responds to it, as > seen in nginx logs, and then closes the connection. The next > request arrives at 14:08:36.496977 and immediately responded with > RST. > > The patch does not change handling of these 4 cases, except may > be some mostly unrelated event handling changes, so the timing > might be slightly different. As such, I would expect the errors > to be still reproducible with the patch, but probably somewhat > less often. > > Could you please re-check to see if the symptoms still happen with > the patch, at least occasionally? You are totally right about that. I tested the previous patch for more times and can see the RST occasionally with less frequency. > > Overall, after looking into logs and tcpdump you've provided I > tend to think that the only working fix would be to introduce > c->pipeline flag, and force lingering close if there were any > pipelined requests on the connection. > > Below is the patch which implements this approach. Review and > testing appreciated. It can be used either separately or with the > previously provided patch to use posted next events. I've tested the new patch for several times and can confirm the problem is solved. Also, I added a new variable exposing the r->connection->pipeline and can confirm that the flag works as intended. The flag won't be set for normal keep-alive requests, and will only be set for pipelined requests. Thanks again for looking into this issue. > > # HG changeset patch > # User Maxim Dounin > # Date 1674790916 -10800 > # Fri Jan 27 06:41:56 2023 +0300 > # Node ID 784d0fa0b5a0796561642a5a64dc4e9e07592852 > # Parent 4eb1383f6432b034630e6de18739b817f6565c8c > Lingering close for connections with pipelined requests. > > This is expected to help with clients using pipelining with some constant > depth, such as apt[1][2]. > > When downloading many resources, apt uses pipelining with some constant > depth, a number of requests in flight[1][2]. This essentially means that > after receiving a response it sends an additional request to the server, > and this can result in requests arriving to the server at any time. Further, > additional requests are sent one-by-one, and can be easily seen as such > (neither as pipelined, nor followed by pipelined requests). > > The only safe approach to close such connections (for example, when > keepalive_requests is reached) is with lingering. To do so, now nginx > monitors if pipelining was used on the connection, and if it was, closes > the connection with lingering. > > [1] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 > [2] https://mailman.nginx.org/pipermail/nginx-devel/2023-January/ZA2SP5SJU55LHEBCJMFDB2AZVELRLTHI.html > > diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h > --- a/src/core/ngx_connection.h > +++ b/src/core/ngx_connection.h > @@ -172,6 +172,7 @@ struct ngx_connection_s { > unsigned timedout:1; > unsigned error:1; > unsigned destroyed:1; > + unsigned pipeline:1; > > unsigned idle:1; > unsigned reusable:1; > diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c > --- a/src/http/ngx_http_request.c > +++ b/src/http/ngx_http_request.c > @@ -2753,7 +2753,8 @@ ngx_http_finalize_connection(ngx_http_re > || (clcf->lingering_close == NGX_HTTP_LINGERING_ON > && (r->lingering_close > || r->header_in->pos < r->header_in->last > - || r->connection->read->ready))) > + || r->connection->read->ready > + || r->connection->pipeline))) > { > ngx_http_set_lingering_close(r->connection); > return; > @@ -3123,6 +3124,7 @@ ngx_http_set_keepalive(ngx_http_request_ > > c->sent = 0; > c->destroyed = 0; > + c->pipeline = 1; > > if (rev->timer_set) { > ngx_del_timer(rev); > > -- > Maxim Dounin > http://mdounin.ru/ Cheers, Miao Wang From mdounin at mdounin.ru Fri Jan 27 21:57:11 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sat, 28 Jan 2023 00:57:11 +0300 Subject: [PATCH] HTTP: trigger lingering close when keepalive connection will be closed In-Reply-To: <01045AAC-62AA-4DEB-84D0-9C65754FD639@gmail.com> References: <699749D2-D96F-47D8-85DA-74E7F97682FF@gmail.com> <8D59E87C-2DC3-43AB-9067-019CA9ACE5C4@gmail.com> <01045AAC-62AA-4DEB-84D0-9C65754FD639@gmail.com> Message-ID: Hello! On Fri, Jan 27, 2023 at 08:11:34PM +0800, Miao Wang wrote: > > 2023年1月27日 12:01，Maxim Dounin 写道： > > > > On Wed, Jan 25, 2023 at 12:49:10PM +0800, Miao Wang wrote: > > > >>> 2023年1月25日 10:17，Maxim Dounin 写道： > >>> > >>> On Mon, Jan 23, 2023 at 07:01:16PM +0800, Miao Wang wrote: > >>> > >>>>> 2023年1月23日 12:05，Maxim Dounin 写道： > >>>>> > >>>>> On Wed, Jan 18, 2023 at 11:28:52PM +0800, Miao Wang wrote: > >>>>> > >>>>>> # HG changeset patch > >>>>>> # User Miao Wang > >>>>>> # Date 1674055068 -28800 > >>>>>> # Wed Jan 18 23:17:48 2023 +0800 > >>>>>> # Node ID 73aa64bd29f3dec9e43e97560d6b5a07cdf40063 > >>>>>> # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > >>>>>> HTTP: trigger lingering close when keepalive connection will be closed > >>>>>> > >>>>>> When finalizing a request, if the request is not keepalive but > >>>>>> its connection has served more than one request, then the connection > >>>>>> has been a keepalive connection previously and this connection will > >>>>>> be closed after this response. In this condition, it is likely that > >>>>>> there are pipelined requests following this request, which we should > >>>>>> ignore. As a result, lingering close is necessary in this case. > >>>>>> > >>>>>> Without this patch, nginx (with its default configuration) will send > >>>>>> out TCP RST when there are more pipelined requests. The symptom is > >>>>>> obvious when nginx is serving a debian repository and apt is > >>>>>> downloading massive of packages. See [1]. It becomes more obvious > >>>>>> when `keepalive_requests` is lower or nginx is under a relative > >>>>>> higher load, and it disappears when specifying > >>>>>> `lingering_close always`. > >>>>>> > >>>>>> [1]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 > >>>>>> > >>>>>> diff -r 07b0bee87f32 -r 73aa64bd29f3 src/http/ngx_http_request.c > >>>>>> --- a/src/http/ngx_http_request.c Wed Dec 21 14:53:27 2022 +0300 > >>>>>> +++ b/src/http/ngx_http_request.c Wed Jan 18 23:17:48 2023 +0800 > >>>>>> @@ -2749,6 +2749,10 @@ > >>>>>> return; > >>>>>> } > >>>>>> > >>>>>> + if (!r->keepalive && r->connection->requests > 1) { > >>>>>> + r->lingering_close = 1; > >>>>>> + } > >>>>>> + > >>>>>> if (clcf->lingering_close == NGX_HTTP_LINGERING_ALWAYS > >>>>>> || (clcf->lingering_close == NGX_HTTP_LINGERING_ON > >>>>>> && (r->lingering_close > >>>>> > >>>>> Thanks for the patch and the link to the Debian bug report. > >>>>> > >>>>> Lingering close implies noticeable additional resource usage: even > >>>>> if nothing happens on the connection, it will be kept open for > >>>>> lingering_timeout, which is 5 seconds by default. Given that > >>>>> pipelining is not used by most of the clients, forcing lingering > >>>>> close for all clients which are using keepalive does not look like > >>>>> a good solution to me. > >>>>> > >>>>> In general, nginx tries hard to determine if any additional data > >>>>> are expected on the connection, and uses lingering close if there > >>>>> is a good chance there will be some, but avoids lingering close by > >>>>> default if additional data are unlikely. If this logic does not > >>>>> work for some reason, lingering close can be explicitly requested > >>>>> with "lingering_close always;". > >>>> > >>>> That's true since the symptom I described can be worked around with > >>>> that option. > >>>> > >>>>> > >>>>> In particular, note the "r->header_in->pos < r->header_in->last" > >>>>> and "r->connection->read->ready" checks - these are expected to > >>>>> catch connections with additional pipelined requests (see revision > >>>>> 3981:77604e9a1ed8). And from the log provided in the report it > >>>>> looks like it works most of the time - there are more than 6k HTTP > >>>>> requests, and 60+ connections. But sometimes it fails - there are > >>>>> two RST errors logged (and one "Undetermined Error", which looks > >>>>> like a bug in apt, but might be related). > >>>>> > >>>>> It looks like when apt is downloading many resources, it does not > >>>>> send all the requests at once (or in batches), but instead tries > >>>>> to maintain a constant "depth", a number of pipelined requests in > >>>>> flight. This essentially means that after reading of a response > >>>>> it sends an additional request. > >>>> > >>>> That's right. From a traffic dump, I can see apt first sends one > >>>> request, and after receiving the response, it will send out 10 > >>>> more requests, and maintain a depth of 10, since by default > >>>> Acquire::http::Pipeline-Depth is 10. > >>>> > >>>>> > >>>>> I see at least two possible cases which can result in nginx not > >>>>> using lingering close with such a load: > >>>>> > >>>>> 1. If a response where keepalive_requests is reached happens to > >>>>> be the last request in the r->header_in buffer (so the > >>>>> "r->header_in->pos < r->header_in->last" won't be effective), and > >>>>> there is a chance that nginx wasn't yet got an event from kernel > >>>>> about additional data (and therefore "r->connection->read->ready" > >>>>> will not be set). As such, nginx won't use lingering close, and > >>>>> might close connection with unread data in the socket buffer, > >>>>> resulting in RST. > >>>>> > >>>>> 2. Similarly, if nginx happens to be faster than apt, and socket > >>>>> buffers are large enough, it might sent all the responses, > >>>>> including the last one with "Connection: close", and close the > >>>>> connection (since there are no pending pipelined requests at the > >>>>> moment) even before an additional request is sent by apt. When > >>>>> later apt will send an additional request after reading some of > >>>>> the responses, it will send the request to already closed > >>>>> connection, again resulting in RST. > >>>> > >>>> Actually, comparing the debug log and the pcap, nginx calls > >>>> close() after writing the last response. However, at that time, > >>>> that response is not fully transmitted to the client and there > >>>> seems to be more requests not processed in the kernel buffer. > >>>> Thus close() triggers an immediate RST. > >>> > >>> Thanks for the details. This looks more like the first case, and > >>> probably can be addressed by improving likelihood of detecting the > >>> read event. > >>> > >>> Could you please test if the patch below fixes the particular > >>> issue you are seeing? It is somewhat unrelated, but it might be > >>> a good enough solution (and is more or less equivalent to > >>> checking r->pipeline). > >>> > >>>>> It would be interesting to see more details, such as tcpdump > >>>>> and/or nginx debug logs, to find out what actually goes on here. > >>>> > >>>> The tcpdump and debug logs are too large to send in this mail list. > >>>> I wonder if I can directly email it to you. > >>> > >>> Feel free to, my email should accept up to 100M messages. > >>> Alternatively, a good solution might be to make the files > >>> available for download and post a link here. > >>> > >>>>> Overall, given how apt uses pipelining, I tend to think that at > >>>>> least (2) is unavoidable and can happen with certain sizes of the > >>>>> responses. > >>>>> > >>>>> A good enough solution might be check for r->pipeline, which is > >>>>> set by nginx as long as it reads a pipelined request. It might > >>>>> not be enough though, since r->pipeline is only set for requests > >>>>> seen by nginx as pipelined, which might not be true for the last > >>>>> request. > >>>>> > >>>>> A more complete solution might be to introduce something like > >>>>> c->pipeline flag and use lingering close if any pipelined requests > >>>>> were seen on the connection. > >>> > >>> The following patch reworks handling of pipelined requests by > >>> postponing them to the next event loop iteration. It is expected > >>> make it more likely for nginx to know there are any additional > >>> unread data in the socket buffer (and right now is mostly > >>> equivalent to checking r->pipeline, since c->read->ready is always > >>> set for pipelined requests): > >>> > >>> # HG changeset patch > >>> # User Maxim Dounin > >>> # Date 1674610218 -10800 > >>> # Wed Jan 25 04:30:18 2023 +0300 > >>> # Node ID 8cfd22c325a3db370b9e45aa6f897ff7bc8222f3 > >>> # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db > >>> Reworked pipelined requests to use posted next events. > >>> > >>> This is expected to improve handling of pipelined requests in a number > >>> of ways, including: > >>> > >>> 1) It will make a room for additional requests from other clients, > >>> reducing worker monopolization by a single client. > >>> > >>> 2) The c->read->ready flag will be set, so nginx will either read the > >>> additional data, or will use lingering close. This is expected to help > >>> with clients using pipelining with some constant depth, such as apt[1][2]. > >>> > >>> The ngx_event_move_posted_next() was modified to make it possible to > >>> post read events on connections with kqueue. Previously, it used to > >>> set ev->available to -1, potentially overwriting a valid positive value > >>> provided by kqueue, so ngx_unix_recv() and ngx_readv_chain() will stop > >>> reading from the socket before reading all the data available. > >>> > >>> Note that currently ngx_event_move_posted_next() will always set > >>> the ev->ready flag. While this is expected behaviour for the ev->available > >>> use case (where ev->ready is explicitly cleared), this is not needed for > >>> pipelining. For pipelining, this will result in extra unneeded read() > >>> syscall after processing of all pipelined requests, and there might be > >>> a room for improvement here. > >>> > >>> [1] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=973861#10 > >>> [2] https://mailman.nginx.org/pipermail/nginx-devel/2023-January/ZA2SP5SJU55LHEBCJMFDB2AZVELRLTHI.html > >>> > >>> diff --git a/src/event/ngx_event_posted.c b/src/event/ngx_event_posted.c > >>> --- a/src/event/ngx_event_posted.c > >>> +++ b/src/event/ngx_event_posted.c > >>> @@ -51,8 +51,10 @@ ngx_event_move_posted_next(ngx_cycle_t * > >>> ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, > >>> "posted next event %p", ev); > >>> > >>> - ev->ready = 1; > >>> - ev->available = -1; > >>> + if (!ev->ready) { > >>> + ev->ready = 1; > >>> + ev->available = -1; > >>> + } > >>> } > >>> > >>> ngx_queue_add(&ngx_posted_events, &ngx_posted_next_events); > >>> diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c > >>> --- a/src/http/ngx_http_request.c > >>> +++ b/src/http/ngx_http_request.c > >>> @@ -3129,7 +3129,7 @@ ngx_http_set_keepalive(ngx_http_request_ > >>> } > >>> > >>> rev->handler = ngx_http_process_request_line; > >>> - ngx_post_event(rev, &ngx_posted_events); > >>> + ngx_post_event(rev, &ngx_posted_next_events); > >>> return; > >>> } > >>> > >> > >> I can confirm that the symptom disappears after applying this patch > > > > Thanks for testing, and thanks for the logs and traffic dumps > > provided. > > > > From the logs you've provided it looks like the patch should help > > with 1 out of 5 errors: the last request in the connection with > > this error was pipelined, and therefore with the patch it is > > expected to use lingering close. > > > > In 4 other cases the last requests are handled without pipelining: > > once the request arrives, ngx_http_keepalive_handler() is called, > > and nginx reads the request and responds to it, closing the > > connection. The next request arrives at some time later, and the > > OS responds with RST. > > > > E.g., the request to bzip2_1.0.8-4_amd64.deb (which corresponds to > > the second error as seen by apt, while downloading > > libpython3.9-stdlib_3.9.2-1_amd64.deb; connection *26, with client > > port 40110) arrives at 14:08:36.495858, nginx responds to it, as > > seen in nginx logs, and then closes the connection. The next > > request arrives at 14:08:36.496977 and immediately responded with > > RST. > > > > The patch does not change handling of these 4 cases, except may > > be some mostly unrelated event handling changes, so the timing > > might be slightly different. As such, I would expect the errors > > to be still reproducible with the patch, but probably somewhat > > less often. > > > > Could you please re-check to see if the symptoms still happen with > > the patch, at least occasionally? > > You are totally right about that. I tested the previous patch > for more times and can see the RST occasionally with less frequency. Thanks for confirming this. > > Overall, after looking into logs and tcpdump you've provided I > > tend to think that the only working fix would be to introduce > > c->pipeline flag, and force lingering close if there were any > > pipelined requests on the connection. > > > > Below is the patch which implements this approach. Review and > > testing appreciated. It can be used either separately or with the > > previously provided patch to use posted next events. > > I've tested the new patch for several times and can confirm the > problem is solved. Also, I added a new variable exposing the > r->connection->pipeline and can confirm that the flag works > as intended. The flag won't be set for normal keep-alive requests, > and will only be set for pipelined requests. > > Thanks again for looking into this issue. Thanks for testing. -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Sat Jan 28 00:15:38 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sat, 28 Jan 2023 03:15:38 +0300 Subject: [PATCH] Added warning about redefinition of listen socket protocol options In-Reply-To: References: <5450B60D-DD89-4861-BA4D-6B57F085F7FD@nginx.com> Message-ID: Hello! On Thu, Jan 26, 2023 at 08:00:53PM +0400, Sergey Kandaurov wrote: > > On 26 Jan 2023, at 01:29, Maxim Dounin wrote: > > > > On Mon, Jan 23, 2023 at 02:21:33PM +0400, Sergey Kandaurov wrote: > > > >>> On 31 Dec 2022, at 18:35, Maxim Dounin wrote: > >>> > >>> # HG changeset patch > >>> # User Maxim Dounin > >>> # Date 1672497248 -10800 > >>> # Sat Dec 31 17:34:08 2022 +0300 > >>> # Node ID c215d5cf25732ece1819cf1cd48ebb480bb642c7 > >>> # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > >>> Added warning about redefinition of listen socket protocol options. > >>> > >>> The "listen" directive in the http module can be used multiple times > >>> in different server blocks. Originally, it was supposed to be specified > >>> once with various socket options, and without any parameters in virtual > >>> server blocks. For example: > >>> > >>> server { listen 80 backlog=1024; server_name foo; ... } > >>> server { listen 80; server_name bar; ... } > >>> server { listen 80; server_name bazz; ... } > >>> > >>> The address part of the syntax ("address[:port]" / "port" / "unix:path") > >>> uniquely identifies the listening socket, and therefore is enough for > >>> name-based virtual servers (to let nginx know that the virtual server > >>> accepts requests on the listening socket in question). > >>> > >>> To ensure that listening options do not conflict between virtual servers, > >>> they were allowed only once. For example, the following configuration > >>> will be rejected ("duplicate listen options for 0.0.0.0:80 in ..."): > >>> > >>> server { listen 80 backlog=1024; server_name foo; ... } > >>> server { listen 80 backlog=512; server_name bar; ... } > >>> > >>> At some point it was, however, noticed, that it is sometimes convenient > >>> to repeat some options for clarity. In nginx 0.8.51 the "ssl" parameter > >>> was allowed to be specified multiple times, e.g.: > >>> > >>> server { listen 443 ssl backlog=1024; server_name foo; ... } > >>> server { listen 443 ssl; server_name bar; ... } > >>> server { listen 443 ssl; server_name bazz; ... } > >>> > >>> This approach makes configuration more readable, since SSL sockets are > >>> immediately visible in the configuration. If this is not needed, just the > >>> address can still be used. > >>> > >>> Later, additional protocol-specific options similar to "ssl" were > >>> introduced, notably "http2" and "proxy_protocol". With these options, > >>> one can write: > >>> > >>> server { listen 443 ssl backlog=1024; server_name foo; ... } > >>> server { listen 443 http2; server_name bar; ... } > >>> server { listen 443 proxy_protocol; server_name bazz; ... } > >>> > >>> The resulting socket will use ssl, http2, and proxy_protocol, but this > >>> is not really obvious from the configuration. > >>> > >>> To ensure such misleading configurations are not allowed, nginx now > >>> warns as long as the "listen" directive is used with options different > >> > >> nitpicking: > >> > >> "ensure .. are not allowed" and "warns" don't seem to be equally strong. > >> As such, I'd rewrite to something like: > >> > >> To emphasize such misleading configurations are discouraged, nginx now > > > > Thanks, changed. > > > >>> from the options previously used if these are potentially confusing. > >> > >> Not really confident what "these" refers to. > >> > >> s/these/they/ ? > > > > I don't think that there is much difference between "these" and > > "they" when it comes to what they refer to. Either way, "if this > > is potentially confusing" is probably better and unambiguously > > refers to the fact that the "listen" directive is used with the > > different options. Changed. > > Sounds good as well, tnx. > > > > >>> In particular, the following configurations are allowed: > >>> > >>> server { listen 8401 ssl backlog=1024; server_name foo; } > >>> server { listen 8401 ssl; server_name bar; } > >>> server { listen 8401 ssl; server_name bazz; } > >>> > >>> server { listen 8402 ssl http2 backlog=1024; server_name foo; } > >>> server { listen 8402 ssl; server_name bar; } > >>> server { listen 8402 ssl; server_name bazz; } > >>> > >>> server { listen 8403 ssl; server_name bar; } > >>> server { listen 8403 ssl; server_name bazz; } > >>> server { listen 8403 ssl http2; server_name foo; } > >>> > >>> server { listen 8404 ssl http2 backlog=1024; server_name foo; } > >>> server { listen 8404 http2; server_name bar; } > >>> server { listen 8404 http2; server_name bazz; } > >>> > >>> server { listen 8405 ssl http2 backlog=1024; server_name foo; } > >>> server { listen 8405 ssl http2; server_name bar; } > >>> server { listen 8405 ssl http2; server_name bazz; } > >>> > >>> server { listen 8406 ssl; server_name foo; } > >>> server { listen 8406; server_name bar; } > >>> server { listen 8406; server_name bazz; } > >>> > >>> And the following configurations will generate warnings: > >>> > >>> server { listen 8501 ssl http2 backlog=1024; server_name foo; } > >>> server { listen 8501 http2; server_name bar; } > >>> server { listen 8501 ssl; server_name bazz; } > >>> > >>> server { listen 8502 backlog=1024; server_name foo; } > >>> server { listen 8502 ssl; server_name bar; } > >>> > >>> server { listen 8503 ssl; server_name foo; } > >>> server { listen 8503 http2; server_name bar; } > >>> > >>> server { listen 8504 ssl; server_name foo; } > >>> server { listen 8504 http2; server_name bar; } > >>> server { listen 8504 proxy_protocol; server_name bazz; } > >>> > >>> server { listen 8505 ssl http2 proxy_protocol; server_name foo; } > >>> server { listen 8505 ssl http2; server_name bar; } > >>> server { listen 8505 ssl; server_name bazz; } > >>> > >>> server { listen 8506 ssl http2; server_name foo; } > >>> server { listen 8506 ssl; server_name bar; } > >>> server { listen 8506; server_name bazz; } > >>> > >>> server { listen 8507 ssl; server_name bar; } > >>> server { listen 8507; server_name bazz; } > >>> server { listen 8507 ssl http2; server_name foo; } > >>> > >>> server { listen 8508 ssl; server_name bar; } > >>> server { listen 8508; server_name bazz; } > >>> server { listen 8508 ssl backlog=1024; server_name foo; } > >>> > >>> server { listen 8509; server_name bazz; } > >>> server { listen 8509 ssl; server_name bar; } > >>> server { listen 8509 ssl backlog=1024; server_name foo; } > >>> > >> > >> 15 examples of dos and don'ts looks slightly excessive. > >> The accurate description (such as provided by you below) allows > >> to reduce most of them to e.g. four common invalid configurations: > >> > >> A lesser option set with socket option: > >> > >> server { listen 8443 backlog=1024; server_name foo; } > >> server { listen 8443 http2; server_name bar; } > >> > >> The main option set is repeated at least twice: > >> > >> server { listen 127.0.0.1:8443; server_name foo; } > >> server { listen 127.0.0.1:8443 ssl; server_name bar; } > >> server { listen 127.0.0.1:8443 ssl; server_name baz; } > >> > >> Option sets partially overlap: > >> > >> server { listen 127.0.0.1:8443 ssl; server_name foo; } > >> server { listen 127.0.0.1:8443 http2; server_name bar; } > >> > >> More than two option sets: > >> > >> server { listen 127.0.0.1:8443 http2 ssl; server_name foo; } > >> server { listen 127.0.0.1:8443 http2; server_name bar; } > >> server { listen 127.0.0.1:8443 ssl; server_name baz; } > > > > While your approach might be better from documentation point > > of view, the way it is currently described in the commit log is > > how it was designed: from the examples of valid and invalid > > configurations. > > > > My current working tests contain 18 valid and 22 invalid > > configurations, derived from the ones provided in the commit log > > with additional shuffling. But since these are derived I've > > decided to avoid providing all of them in the commit log. > > > >>> The basic idea is that at most two sets of protocol options are allowed: > >>> the main one (with socket options, if any), and a shorter one, with options > >>> being a subset of the main options, repeated for clarity. As long as the > >>> shorter set of protocol options is used, all listen directives except the > >>> main one should use it. > >> > >> I'd move this paragraph somewhere before examples, as this is the most > >> specific description of things actually changed. > > > > This paragraph summarizes changes made to address the issue > > described, and I don't think moving it will improve things. > > The above looks sane now, thanks for the explanation. > I won't insist on further adjustments. > > > > >> BTW, while reviewing I caught sort of a bug. > >> As I understand the above explanation, if there are both full and short > >> sets present, then at most one listen directive can have the full set, > >> while shorter sets can be repeated. If so, then with the proposed patch > >> the next configuration is expectedly invalid: > >> > >> server { listen 127.0.0.1:8443 ssl; server_name foo; } > >> server { listen 127.0.0.1:8443 ssl; server_name bar; } > >> server { listen 127.0.0.1:8443; server_name baz; } > >> > >> This is expected since first two servers with same options are > >> interpreted as a short form (with full form seen potentially later on), > >> but 3rd server has lesser options, which is caught by this check: > >> (addr[i].protocols_set && protocols != addr[i].protocols) > >> Which is interpreted as: > >> "this server has a lesser set that doesn't match a a shorter set". > >> > >> Now, if 3rd server is moved first, configuration starts to pass: > >> > >> server { listen 127.0.0.1:8443; server_name baz; } > >> server { listen 127.0.0.1:8443 ssl; server_name foo; } > >> server { listen 127.0.0.1:8443 ssl; server_name bar; } > >> > >> This is because after (now) 2nd server, it is parsed as: > >> 1st server has a short form, and 2nd server has a full form. > >> Then 3rd server goes to "the same options" case. This also > >> overwrites the remembered shorter set in addr[i].protocols. > >> > >> I guess an additional check should be added to address this. > >> It is similar to the "options removed" case and ensures that > >> the repeated options set actually matches a shorter set: > >> > >> diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c > >> --- a/src/http/ngx_http.c > >> +++ b/src/http/ngx_http.c > >> @@ -1345,7 +1345,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, n > >> > >> /* the same options */ > >> > >> - if (lsopt->set && addr[i].protocols_changed) { > >> + if ((lsopt->set && addr[i].protocols_changed) > >> + || (addr[i].protocols_set && protocols != addr[i].protocols)) > >> + { > >> ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > >> "protocol options redefined for %V", > >> &addr[i].opt.addr_text); > > > > Thanks for catching this, applied. > > > > Updated patch: > > > > # HG changeset patch > > # User Maxim Dounin > > # Date 1674624978 -10800 > > # Wed Jan 25 08:36:18 2023 +0300 > > # Node ID 1619d1563c3231d4283937610ce97aa1e3824fb8 > > # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db > > Added warning about redefinition of listen socket protocol options. > > > > The "listen" directive in the http module can be used multiple times > > in different server blocks. Originally, it was supposed to be specified > > once with various socket options, and without any parameters in virtual > > server blocks. For example: > > > > server { listen 80 backlog=1024; server_name foo; ... } > > server { listen 80; server_name bar; ... } > > server { listen 80; server_name bazz; ... } > > > > The address part of the syntax ("address[:port]" / "port" / "unix:path") > > uniquely identifies the listening socket, and therefore is enough for > > name-based virtual servers (to let nginx know that the virtual server > > accepts requests on the listening socket in question). > > > > To ensure that listening options do not conflict between virtual servers, > > they were allowed only once. For example, the following configuration > > will be rejected ("duplicate listen options for 0.0.0.0:80 in ..."): > > > > server { listen 80 backlog=1024; server_name foo; ... } > > server { listen 80 backlog=512; server_name bar; ... } > > > > At some point it was, however, noticed, that it is sometimes convenient > > to repeat some options for clarity. In nginx 0.8.51 the "ssl" parameter > > was allowed to be specified multiple times, e.g.: > > > > server { listen 443 ssl backlog=1024; server_name foo; ... } > > server { listen 443 ssl; server_name bar; ... } > > server { listen 443 ssl; server_name bazz; ... } > > > > This approach makes configuration more readable, since SSL sockets are > > immediately visible in the configuration. If this is not needed, just the > > address can still be used. > > > > Later, additional protocol-specific options similar to "ssl" were > > introduced, notably "http2" and "proxy_protocol". With these options, > > one can write: > > > > server { listen 443 ssl backlog=1024; server_name foo; ... } > > server { listen 443 http2; server_name bar; ... } > > server { listen 443 proxy_protocol; server_name bazz; ... } > > > > The resulting socket will use ssl, http2, and proxy_protocol, but this > > is not really obvious from the configuration. > > > > To emphasize such misleading configurations are discouraged, nginx now > > warns as long as the "listen" directive is used with options different > > from the options previously used if this is potentially confusing. > > > > In particular, the following configurations are allowed: > > > > server { listen 8401 ssl backlog=1024; server_name foo; } > > server { listen 8401 ssl; server_name bar; } > > server { listen 8401 ssl; server_name bazz; } > > > > server { listen 8402 ssl http2 backlog=1024; server_name foo; } > > server { listen 8402 ssl; server_name bar; } > > server { listen 8402 ssl; server_name bazz; } > > > > server { listen 8403 ssl; server_name bar; } > > server { listen 8403 ssl; server_name bazz; } > > server { listen 8403 ssl http2; server_name foo; } > > > > server { listen 8404 ssl http2 backlog=1024; server_name foo; } > > server { listen 8404 http2; server_name bar; } > > server { listen 8404 http2; server_name bazz; } > > > > server { listen 8405 ssl http2 backlog=1024; server_name foo; } > > server { listen 8405 ssl http2; server_name bar; } > > server { listen 8405 ssl http2; server_name bazz; } > > > > server { listen 8406 ssl; server_name foo; } > > server { listen 8406; server_name bar; } > > server { listen 8406; server_name bazz; } > > > > And the following configurations will generate warnings: > > > > server { listen 8501 ssl http2 backlog=1024; server_name foo; } > > server { listen 8501 http2; server_name bar; } > > server { listen 8501 ssl; server_name bazz; } > > > > server { listen 8502 backlog=1024; server_name foo; } > > server { listen 8502 ssl; server_name bar; } > > > > server { listen 8503 ssl; server_name foo; } > > server { listen 8503 http2; server_name bar; } > > > > server { listen 8504 ssl; server_name foo; } > > server { listen 8504 http2; server_name bar; } > > server { listen 8504 proxy_protocol; server_name bazz; } > > > > server { listen 8505 ssl http2 proxy_protocol; server_name foo; } > > server { listen 8505 ssl http2; server_name bar; } > > server { listen 8505 ssl; server_name bazz; } > > > > server { listen 8506 ssl http2; server_name foo; } > > server { listen 8506 ssl; server_name bar; } > > server { listen 8506; server_name bazz; } > > > > server { listen 8507 ssl; server_name bar; } > > server { listen 8507; server_name bazz; } > > server { listen 8507 ssl http2; server_name foo; } > > > > server { listen 8508 ssl; server_name bar; } > > server { listen 8508; server_name bazz; } > > server { listen 8508 ssl backlog=1024; server_name foo; } > > > > server { listen 8509; server_name bazz; } > > server { listen 8509 ssl; server_name bar; } > > server { listen 8509 ssl backlog=1024; server_name foo; } > > > > The basic idea is that at most two sets of protocol options are allowed: > > the main one (with socket options, if any), and a shorter one, with options > > being a subset of the main options, repeated for clarity. As long as the > > shorter set of protocol options is used, all listen directives except the > > main one should use it. > > > > diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c > > --- a/src/http/ngx_http.c > > +++ b/src/http/ngx_http.c > > @@ -1228,7 +1228,8 @@ static ngx_int_t > > ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, > > ngx_http_conf_port_t *port, ngx_http_listen_opt_t *lsopt) > > { > > - ngx_uint_t i, default_server, proxy_protocol; > > + ngx_uint_t i, default_server, proxy_protocol, > > + protocols, protocols_prev; > > ngx_http_conf_addr_t *addr; > > #if (NGX_HTTP_SSL) > > ngx_uint_t ssl; > > @@ -1264,12 +1265,18 @@ ngx_http_add_addresses(ngx_conf_t *cf, n > > default_server = addr[i].opt.default_server; > > > > proxy_protocol = lsopt->proxy_protocol || addr[i].opt.proxy_protocol; > > + protocols = lsopt->proxy_protocol; > > + protocols_prev = addr[i].opt.proxy_protocol; > > > > #if (NGX_HTTP_SSL) > > ssl = lsopt->ssl || addr[i].opt.ssl; > > + protocols |= lsopt->ssl << 1; > > + protocols_prev |= addr[i].opt.ssl << 1; > > #endif > > #if (NGX_HTTP_V2) > > http2 = lsopt->http2 || addr[i].opt.http2; > > + protocols |= lsopt->http2 << 2; > > + protocols_prev |= addr[i].opt.http2 << 2; > > #endif > > > > if (lsopt->set) { > > @@ -1299,6 +1306,57 @@ ngx_http_add_addresses(ngx_conf_t *cf, n > > addr[i].default_server = cscf; > > } > > > > + /* check for conflicting protocol options */ > > + > > + if ((protocols | protocols_prev) != protocols_prev) { > > + > > + /* options added */ > > + > > + if ((addr[i].opt.set && !lsopt->set) > > + || addr[i].protocols_changed > > + || (protocols | protocols_prev) != protocols) > > + { > > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > > + "protocol options redefined for %V", > > + &addr[i].opt.addr_text); > > + } > > + > > + addr[i].protocols = protocols_prev; > > + addr[i].protocols_set = 1; > > + addr[i].protocols_changed = 1; > > + > > + } else if ((protocols_prev | protocols) != protocols) { > > + > > + /* options removed */ > > + > > + if (lsopt->set > > + || (addr[i].protocols_set && protocols != addr[i].protocols)) > > + { > > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > > + "protocol options redefined for %V", > > + &addr[i].opt.addr_text); > > + } > > + > > + addr[i].protocols = protocols; > > + addr[i].protocols_set = 1; > > + addr[i].protocols_changed = 1; > > + > > + } else { > > + > > + /* the same options */ > > + > > + if ((lsopt->set && addr[i].protocols_changed) > > + || (addr[i].protocols_set && protocols != addr[i].protocols)) > > + { > > + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, > > + "protocol options redefined for %V", > > + &addr[i].opt.addr_text); > > + } > > + > > + addr[i].protocols = protocols; > > + addr[i].protocols_set = 1; > > + } > > + > > addr[i].opt.default_server = default_server; > > addr[i].opt.proxy_protocol = proxy_protocol; > > #if (NGX_HTTP_SSL) > > @@ -1355,6 +1413,9 @@ ngx_http_add_address(ngx_conf_t *cf, ngx > > } > > > > addr->opt = *lsopt; > > + addr->protocols = 0; > > + addr->protocols_set = 0; > > + addr->protocols_changed = 0; > > addr->hash.buckets = NULL; > > addr->hash.size = 0; > > addr->wc_head = NULL; > > diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h > > --- a/src/http/ngx_http_core_module.h > > +++ b/src/http/ngx_http_core_module.h > > @@ -274,6 +274,10 @@ typedef struct { > > typedef struct { > > ngx_http_listen_opt_t opt; > > > > + unsigned protocols:3; > > + unsigned protocols_set:1; > > + unsigned protocols_changed:1; > > + > > ngx_hash_t hash; > > ngx_hash_wildcard_t *wc_head; > > ngx_hash_wildcard_t *wc_tail; > > > > Looks good now, thanks. Thanks for the review, pushed to http://mdounin.ru/hg/nginx. Also, a fix to the only warning generated by tests is pushed to http://mdounin.ru/hg/nginx-tests. -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Sat Jan 28 20:03:17 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sat, 28 Jan 2023 23:03:17 +0300 Subject: [PATCH] Gzip static: ranges support (ticket #2349) In-Reply-To: References: <262ABC58-7AB2-4B53-90D4-648464DE4F44@nginx.com> Message-ID: Hello! On Wed, Jan 25, 2023 at 01:50:56PM +0400, Sergey Kandaurov wrote: > > On 24 Jan 2023, at 06:19, Maxim Dounin wrote: > > > > On Mon, Jan 23, 2023 at 09:28:42PM +0400, Sergey Kandaurov wrote: > > > >>> [..] > > > >> On a related note, while comparing with static module, which gzip_static > >> is based on, I further noticed that gzip_static doesn't check for 0-size > >> response in subrequests. Existing handling of r->main suggests that > >> such configuration might be used in practice, e.g. together with gunzip > >> filter, as documented in the gzip_static module documentation. > >> So, it makes sense to add such check for zero size buffers as well. > >> > >> # HG changeset patch > >> # User Sergey Kandaurov > >> # Date 1674493925 -14400 > >> # Mon Jan 23 21:12:05 2023 +0400 > >> # Node ID 27217fca1966ddb20c843384d438df2af062fdfc > >> # Parent dd458c69858b88231f542be4573a3f81141d1359 > >> Gzip static: avoid "zero size buf" alerts in subrequests. > >> > >> Similar to the static module, gzip_static enabled in subrequests might > >> result in zero size buffers with responses from empty precompressed files. > > > > This looks like an issue introduced in 4611:2b6cb7528409, and it > > might be a good idea to fix other affected modules as well. > > > > While valid gzipped responses are not expected to be empty, much > > like valid mp4 and flv files, but it certainly shouldn't cause > > "zero size buf" alerts if there is an invalid file for some > > reason. > > > >> diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c > >> --- a/src/http/modules/ngx_http_gzip_static_module.c > >> +++ b/src/http/modules/ngx_http_gzip_static_module.c > >> @@ -236,6 +236,10 @@ ngx_http_gzip_static_handler(ngx_http_re > >> return NGX_HTTP_INTERNAL_SERVER_ERROR; > >> } > >> > >> + if (r != r->main && of.size == 0) { > >> + return ngx_http_send_header(r); > >> + } > >> + > >> h = ngx_list_push(&r->headers_out.headers); > >> if (h == NULL) { > >> return NGX_HTTP_INTERNAL_SERVER_ERROR; > > > > While there seems to be no practical difference, I don't think > > that not adding the "Content-Encoding" header for empty subrequest > > responses is a good idea. > > I don't see either. Since it's served by the gzip_static handler, > I tend to agree, in a common sense, to always attach this header. > > > > > Further, I don't actually think that skipping sending the body, as > > the static module currently does, is a good idea either. It might > > break various filter modules which might not expect such > > behaviour. > > > > I would rather consider sending the body as usual, but with > > b->sync set when sending an empty body in subrequests, similarly > > to how ngx_http_send_special() does. Something like this will do > > the trick: > > > > diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c > > --- a/src/http/modules/ngx_http_gzip_static_module.c > > +++ b/src/http/modules/ngx_http_gzip_static_module.c > > @@ -273,6 +273,7 @@ ngx_http_gzip_static_handler(ngx_http_re > > b->in_file = b->file_last ? 1 : 0; > > b->last_buf = (r == r->main) ? 1 : 0; > > b->last_in_chain = 1; > > + b->sync = (r == r->main || b->file_last) ? 0 : 1; > > > > b->file->fd = of.fd; > > b->file->name = path; > > > > In particular, with this patch inflate() is able to properly > > report errors on empty files: > > > > 2023/01/24 03:54:44 [error] 22886#100160: *1 inflate() returned -5 on response end while sending response to client, client: 127.0.0.1, server: , request: "GET /index.html HTTP/1.1", subrequest: "/empty.html", host: "127.0.0.1:8080" > > Note that if error is returned by any filter in subrequest, such as > with gunzip, this terminates the whole request, due to c->error set. > Well, this behaviour looks good enough, compared to half-baked responses. > > > > > I also observe a similar behaviour change with empty xml files > > returned in subrequests by static module and processed by the xslt > > filter: previously, empty xml files were effectively ignored, and > > now they result in the errors much like any other malformed xml > > files. > > Well, that confirms that other such places are susceptible. > > > > > Full patch: > > > > # HG changeset patch > > # User Maxim Dounin > > # Date 1674526244 -10800 > > # Tue Jan 24 05:10:44 2023 +0300 > > # Node ID 22e41eed77ad95f51d7d0a3daa5cb369c9643697 > > # Parent c7e103acb409f0352cb73997c053b3bdbb8dd5db > > Fixed "zero size buf" alerts with subrequests. > > > > Since 4611:2b6cb7528409 responses from the gzip static, flv, and mp4 modules > > can be used with subrequests, though empty files were not properly handled. > > Empty gzipped, flv, and mp4 files thus resulted in "zero size buf in output" > > alerts. While valid corresponding files are not expected to be empty, such > > files shouldn't result in alerts. > > > > Fix is to set b->sync on such empty subrequest responses, similarly to what > > ngx_http_send_special() does. > > > > Additionally, the static module is modified to do the same instead of not > > sending the response body at all in such cases, since not sending the response > > body at all is believed to be at least questionable, and might break various > > filters which do not expect such behaviour. > > > > diff --git a/src/http/modules/ngx_http_flv_module.c b/src/http/modules/ngx_http_flv_module.c > > --- a/src/http/modules/ngx_http_flv_module.c > > +++ b/src/http/modules/ngx_http_flv_module.c > > @@ -235,6 +235,7 @@ ngx_http_flv_handler(ngx_http_request_t > > b->in_file = b->file_last ? 1: 0; > > btw, there are old style issues originated in the static module > (it was fixed in gzip_static when it was copied from there) This is actually an (old) style, which was previously used in many places. Now it remains mostly in multi-line forms of the conditional operator, e.g.: r->loc_conf = (node->exact) ? node->exact->loc_conf: node->inclusive->loc_conf; Single-line form is indeed mostly unused now, and probably it's time to remove remaining occurrences. Added a patch for this. [...] > > @@ -238,10 +238,6 @@ ngx_http_static_handler(ngx_http_request > > return NGX_HTTP_INTERNAL_SERVER_ERROR; > > } > > > > - if (r != r->main && of.size == 0) { > > - return ngx_http_send_header(r); > > - } > > - > > r->allow_ranges = 1; > > > > /* we need to allocate all before the header would be sent */ > > @@ -268,6 +264,7 @@ ngx_http_static_handler(ngx_http_request > > b->in_file = b->file_last ? 1: 0; > > b->last_buf = (r == r->main) ? 1: 0; > > b->last_in_chain = 1; > > + b->sync = (r == r->main || b->file_last) ? 0 : 1; > > > > b->file->fd = of.fd; > > b->file->name = path; > > > > In addition to the static module change, any reason not to include > ngx_http_cache_send()? It has a similar behaviour to skip cached > empty body if served in subrequests. Yet another similar place is > subrequests processed with rewrite such as "return 204;". Indeed, thanks for catching, missed these two. Also, looking into this places I tend to think it's better to use b->sync = (b->last_buf || b->in_file) ? 0 : 1; instead, which is going to be simpler and more universal across all uses. # HG changeset patch # User Maxim Dounin # Date 1674872423 -10800 # Sat Jan 28 05:20:23 2023 +0300 # Node ID 1c3b78d7cdc9e7059e90aa1670d58fd37927a1a2 # Parent 106328a70f4ecb32f828d33e5cd66c861e455f92 Style. diff --git a/src/event/ngx_event_connectex.c b/src/event/ngx_event_connectex.c --- a/src/event/ngx_event_connectex.c +++ b/src/event/ngx_event_connectex.c @@ -127,8 +127,8 @@ void ngx_iocp_wait_events(int main) conn[0] = NULL; for ( ;; ) { - offset = (nevents == WSA_MAXIMUM_WAIT_EVENTS + 1) ? 1: 0; - timeout = (nevents == 1 && !first) ? 60000: INFINITE; + offset = (nevents == WSA_MAXIMUM_WAIT_EVENTS + 1) ? 1 : 0; + timeout = (nevents == 1 && !first) ? 60000 : INFINITE; n = WSAWaitForMultipleEvents(nevents - offset, events[offset], 0, timeout, 0); diff --git a/src/http/modules/ngx_http_flv_module.c b/src/http/modules/ngx_http_flv_module.c --- a/src/http/modules/ngx_http_flv_module.c +++ b/src/http/modules/ngx_http_flv_module.c @@ -232,7 +232,7 @@ ngx_http_flv_handler(ngx_http_request_t b->file_pos = start; b->file_last = of.size; - b->in_file = b->file_last ? 1: 0; + b->in_file = b->file_last ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; diff --git a/src/http/modules/ngx_http_static_module.c b/src/http/modules/ngx_http_static_module.c --- a/src/http/modules/ngx_http_static_module.c +++ b/src/http/modules/ngx_http_static_module.c @@ -265,8 +265,8 @@ ngx_http_static_handler(ngx_http_request b->file_pos = 0; b->file_last = of.size; - b->in_file = b->file_last ? 1: 0; - b->last_buf = (r == r->main) ? 1: 0; + b->in_file = b->file_last ? 1 : 0; + b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; b->file->fd = of.fd; diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -1600,8 +1600,8 @@ ngx_http_cache_send(ngx_http_request_t * b->file_pos = c->body_start; b->file_last = c->length; - b->in_file = (c->length - c->body_start) ? 1: 0; - b->last_buf = (r == r->main) ? 1: 0; + b->in_file = (c->length - c->body_start) ? 1 : 0; + b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; b->file->fd = c->file.fd; # HG changeset patch # User Maxim Dounin # Date 1674872613 -10800 # Sat Jan 28 05:23:33 2023 +0300 # Node ID f5515e727656b2cc529b26a0fe6b4496c62e32b2 # Parent 1c3b78d7cdc9e7059e90aa1670d58fd37927a1a2 Fixed "zero size buf" alerts with subrequests. Since 4611:2b6cb7528409 responses from the gzip static, flv, and mp4 modules can be used with subrequests, though empty files were not properly handled. Empty gzipped, flv, and mp4 files thus resulted in "zero size buf in output" alerts. While valid corresponding files are not expected to be empty, such files shouldn't result in alerts. Fix is to set b->sync on such empty subrequest responses, similarly to what ngx_http_send_special() does. Additionally, the static module, the ngx_http_send_response() function, and file cache are modified to do the same instead of not sending the response body at all in such cases, since not sending the response body at all is believed to be at least questionable, and might break various filters which do not expect such behaviour. diff --git a/src/http/modules/ngx_http_flv_module.c b/src/http/modules/ngx_http_flv_module.c --- a/src/http/modules/ngx_http_flv_module.c +++ b/src/http/modules/ngx_http_flv_module.c @@ -235,6 +235,7 @@ ngx_http_flv_handler(ngx_http_request_t b->in_file = b->file_last ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (b->last_buf || b->in_file) ? 0 : 1; b->file->fd = of.fd; b->file->name = path; diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c --- a/src/http/modules/ngx_http_gzip_static_module.c +++ b/src/http/modules/ngx_http_gzip_static_module.c @@ -273,6 +273,7 @@ ngx_http_gzip_static_handler(ngx_http_re b->in_file = b->file_last ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (b->last_buf || b->in_file) ? 0 : 1; b->file->fd = of.fd; b->file->name = path; diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c --- a/src/http/modules/ngx_http_mp4_module.c +++ b/src/http/modules/ngx_http_mp4_module.c @@ -714,6 +714,7 @@ ngx_http_mp4_handler(ngx_http_request_t b->in_file = b->file_last ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (b->last_buf || b->in_file) ? 0 : 1; b->file->fd = of.fd; b->file->name = path; diff --git a/src/http/modules/ngx_http_static_module.c b/src/http/modules/ngx_http_static_module.c --- a/src/http/modules/ngx_http_static_module.c +++ b/src/http/modules/ngx_http_static_module.c @@ -238,10 +238,6 @@ ngx_http_static_handler(ngx_http_request return NGX_HTTP_INTERNAL_SERVER_ERROR; } - if (r != r->main && of.size == 0) { - return ngx_http_send_header(r); - } - r->allow_ranges = 1; /* we need to allocate all before the header would be sent */ @@ -268,6 +264,7 @@ ngx_http_static_handler(ngx_http_request b->in_file = b->file_last ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (b->last_buf || b->in_file) ? 0 : 1; b->file->fd = of.fd; b->file->name = path; diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -1803,10 +1803,6 @@ ngx_http_send_response(ngx_http_request_ } } - if (r != r->main && val.len == 0) { - return ngx_http_send_header(r); - } - b = ngx_calloc_buf(r->pool); if (b == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; @@ -1817,6 +1813,7 @@ ngx_http_send_response(ngx_http_request_ b->memory = val.len ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (b->last_buf || b->memory) ? 0 : 1; out.buf = b; out.next = NULL; diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -1575,10 +1575,6 @@ ngx_http_cache_send(ngx_http_request_t * ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http file cache send: %s", c->file.name.data); - if (r != r->main && c->length - c->body_start == 0) { - return ngx_http_send_header(r); - } - /* we need to allocate all before the header would be sent */ b = ngx_calloc_buf(r->pool); @@ -1603,6 +1599,7 @@ ngx_http_cache_send(ngx_http_request_t * b->in_file = (c->length - c->body_start) ? 1 : 0; b->last_buf = (r == r->main) ? 1 : 0; b->last_in_chain = 1; + b->sync = (b->last_buf || b->in_file) ? 0 : 1; b->file->fd = c->file.fd; b->file->name = c->file.name; -- Maxim Dounin http://mdounin.ru/ From jordanc.carter at outlook.com Sat Jan 28 23:15:58 2023 From: jordanc.carter at outlook.com (J Carter) Date: Sat, 28 Jan 2023 23:15:58 +0000 Subject: [PATCH] Added dark mode support to error pages - request in ticket #2434 Message-ID: Hello, An example of the dark mode and light mode pages is here https://imgur.com/a/n9QJpp4 # HG changeset patch # User jordanc.carter at outlook.com # Date 1674945436 0 # Sat Jan 28 22:37:16 2023 +0000 # Branch dark-error # Node ID 0dcba21038765f6f03098cbdf23f401e89e3648f # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 Added dark mode support to error pages - request in ticket #2434 diff -r 07b0bee87f32 -r 0dcba2103876 src/http/ngx_http_special_response.c --- a/src/http/ngx_http_special_response.c Wed Dec 21 14:53:27 2022 +0300 +++ b/src/http/ngx_http_special_response.c Sat Jan 28 22:37:16 2023 +0000 @@ -18,6 +18,27 @@ static ngx_int_t ngx_http_send_refresh(ngx_http_request_t *r); +#define ngx_error_page_head(MESSAGE) \ +"" CRLF \ +"" CRLF \ +"" MESSAGE "" CRLF \ +"" CRLF \ +"" CRLF \ +"" CRLF \ +"

" MESSAGE "

" CRLF + + +#define ngx_error_page_head_br(MESSAGE) \ +"" CRLF \ +"" CRLF \ +"400" MESSAGE "" CRLF \ +"" CRLF \ +"" CRLF \ +"" CRLF \ +"

400 Bad Request

" CRLF \ +"
" MESSAGE "
" CRLF + + static u_char ngx_http_error_full_tail[] = "
" NGINX_VER "
" CRLF "" CRLF @@ -57,284 +78,140 @@ "\">" CRLF; -static char ngx_http_error_301_page[] = -"" CRLF -"301 Moved Permanently" CRLF -"" CRLF -"

301 Moved Permanently

" CRLF -; +static char ngx_http_error_301_page[] = + ngx_error_page_head("301 Moved Permanently"); static char ngx_http_error_302_page[] = -"" CRLF -"302 Found" CRLF -"" CRLF -"

302 Found

" CRLF -; + ngx_error_page_head("302 Found"); static char ngx_http_error_303_page[] = -"" CRLF -"303 See Other" CRLF -"" CRLF -"

303 See Other

" CRLF -; + ngx_error_page_head("303 See Other"); static char ngx_http_error_307_page[] = -"" CRLF -"307 Temporary Redirect" CRLF -"" CRLF -"

307 Temporary Redirect

" CRLF -; + ngx_error_page_head("307 Temporary Redirect"); static char ngx_http_error_308_page[] = -"" CRLF -"308 Permanent Redirect" CRLF -"" CRLF -"

308 Permanent Redirect

" CRLF -; + ngx_error_page_head("308 Permanent Redirect"); static char ngx_http_error_400_page[] = -"" CRLF -"400 Bad Request" CRLF -"" CRLF -"

400 Bad Request

" CRLF -; + ngx_error_page_head("400 Bad Request"); static char ngx_http_error_401_page[] = -"" CRLF -"401 Authorization Required" CRLF -"" CRLF -"

401 Authorization Required

" CRLF -; + ngx_error_page_head("401 Authorization Required"); static char ngx_http_error_402_page[] = -"" CRLF -"402 Payment Required" CRLF -"" CRLF -"

402 Payment Required

" CRLF -; + ngx_error_page_head("402 Payment Required"); static char ngx_http_error_403_page[] = -"" CRLF -"403 Forbidden" CRLF -"" CRLF -"

403 Forbidden

" CRLF -; + ngx_error_page_head("403 Forbidden"); static char ngx_http_error_404_page[] = -"" CRLF -"404 Not Found" CRLF -"" CRLF -"

404 Not Found

" CRLF -; + ngx_error_page_head("404 Not Found"); static char ngx_http_error_405_page[] = -"" CRLF -"405 Not Allowed" CRLF -"" CRLF -"

405 Not Allowed

" CRLF -; + ngx_error_page_head("405 Not Allowed"); static char ngx_http_error_406_page[] = -"" CRLF -"406 Not Acceptable" CRLF -"" CRLF -"

406 Not Acceptable

" CRLF -; + ngx_error_page_head("406 Not Acceptable"); static char ngx_http_error_408_page[] = -"" CRLF -"408 Request Time-out" CRLF -"" CRLF -"

408 Request Time-out

" CRLF -; + ngx_error_page_head("408 Request Time-out"); static char ngx_http_error_409_page[] = -"" CRLF -"409 Conflict" CRLF -"" CRLF -"

409 Conflict

" CRLF -; + ngx_error_page_head("409 Conflict"); static char ngx_http_error_410_page[] = -"" CRLF -"410 Gone" CRLF -"" CRLF -"

410 Gone

" CRLF -; + ngx_error_page_head("410 Gone"); static char ngx_http_error_411_page[] = -"" CRLF -"411 Length Required" CRLF -"" CRLF -"

411 Length Required

" CRLF -; + ngx_error_page_head("411 Length Required"); static char ngx_http_error_412_page[] = -"" CRLF -"412 Precondition Failed" CRLF -"" CRLF -"

412 Precondition Failed

" CRLF -; + ngx_error_page_head("412 Precondition Failed"); static char ngx_http_error_413_page[] = -"" CRLF -"413 Request Entity Too Large" CRLF -"" CRLF -"

413 Request Entity Too Large

" CRLF -; + ngx_error_page_head("413 Request Entity Too Large"); static char ngx_http_error_414_page[] = -"" CRLF -"414 Request-URI Too Large" CRLF -"" CRLF -"

414 Request-URI Too Large

" CRLF -; + ngx_error_page_head("414 Request-URI Too Large"); static char ngx_http_error_415_page[] = -"" CRLF -"415 Unsupported Media Type" CRLF -"" CRLF -"

415 Unsupported Media Type

" CRLF -; + ngx_error_page_head("415 Unsupported Media Type"); static char ngx_http_error_416_page[] = -"" CRLF -"416 Requested Range Not Satisfiable" CRLF -"" CRLF -"

416 Requested Range Not Satisfiable

" CRLF -; + ngx_error_page_head("416 Requested Range Not Satisfiable"); static char ngx_http_error_421_page[] = -"" CRLF -"421 Misdirected Request" CRLF -"" CRLF -"

421 Misdirected Request

" CRLF -; + ngx_error_page_head("421 Misdirected Request"); static char ngx_http_error_429_page[] = -"" CRLF -"429 Too Many Requests" CRLF -"" CRLF -"

429 Too Many Requests

" CRLF -; + ngx_error_page_head("429 Too Many Requests"); -static char ngx_http_error_494_page[] = -"" CRLF -"400 Request Header Or Cookie Too Large" -CRLF -"" CRLF -"

400 Bad Request

" CRLF -"
Request Header Or Cookie Too Large
" CRLF -; +static char ngx_http_error_494_page[] = + ngx_error_page_head_br("Request Header Or Cookie Too Large"); -static char ngx_http_error_495_page[] = -"" CRLF -"400 The SSL certificate error" -CRLF -"" CRLF -"

400 Bad Request

" CRLF -"
The SSL certificate error
" CRLF -; +static char ngx_http_error_495_page[] = + ngx_error_page_head_br("The SSL certificate error"); static char ngx_http_error_496_page[] = -"" CRLF -"400 No required SSL certificate was sent" -CRLF -"" CRLF -"

400 Bad Request

" CRLF -"
No required SSL certificate was sent
" CRLF -; + ngx_error_page_head_br("No required SSL certificate was sent"); static char ngx_http_error_497_page[] = -"" CRLF -"400 The plain HTTP request was sent to HTTPS port" -CRLF -"" CRLF -"

400 Bad Request

" CRLF -"
The plain HTTP request was sent to HTTPS port
" CRLF -; + ngx_error_page_head_br("The plain HTTP request was sent to HTTPS port"); static char ngx_http_error_500_page[] = -"" CRLF -"500 Internal Server Error" CRLF -"" CRLF -"

500 Internal Server Error

" CRLF -; + ngx_error_page_head("500 Internal Server Error"); static char ngx_http_error_501_page[] = -"" CRLF -"501 Not Implemented" CRLF -"" CRLF -"

501 Not Implemented

" CRLF -; + ngx_error_page_head("501 Not Implemented"); static char ngx_http_error_502_page[] = -"" CRLF -"502 Bad Gateway" CRLF -"" CRLF -"

502 Bad Gateway

" CRLF -; + ngx_error_page_head("502 Bad Gateway"); static char ngx_http_error_503_page[] = -"" CRLF -"503 Service Temporarily Unavailable" CRLF -"" CRLF -"

503 Service Temporarily Unavailable

" CRLF -; + ngx_error_page_head("503 Service Temporarily Unavailabley"); static char ngx_http_error_504_page[] = -"" CRLF -"504 Gateway Time-out" CRLF -"" CRLF -"

504 Gateway Time-out

" CRLF -; + ngx_error_page_head("504 Gateway Time-out"); static char ngx_http_error_505_page[] = -"" CRLF -"505 HTTP Version Not Supported" CRLF -"" CRLF -"

505 HTTP Version Not Supported

" CRLF -; + ngx_error_page_head("505 HTTP Version Not Supported"); static char ngx_http_error_507_page[] = -"" CRLF -"507 Insufficient Storage" CRLF -"" CRLF -"

507 Insufficient Storage

" CRLF -; + ngx_error_page_head("507 Insufficient Storage"); static ngx_str_t ngx_http_error_pages[] = { From jordanc.carter at outlook.com Sun Jan 29 00:35:07 2023 From: jordanc.carter at outlook.com (J Carter) Date: Sun, 29 Jan 2023 00:35:07 +0000 Subject: [PATCH] Added dark mode support to error pages - request in ticket #2434 In-Reply-To: References: Message-ID: Adding the patch file, as something is messing with the formatting of the patches in email body.. On 28/01/2023 23:15, J Carter wrote: > Hello, > > An example of the dark mode and light mode pages is here > https://imgur.com/a/n9QJpp4 > > # HG changeset patch > # User jordanc.carter at outlook.com > # Date 1674945436 0 > # Sat Jan 28 22:37:16 2023 +0000 > # Branch dark-error > # Node ID 0dcba21038765f6f03098cbdf23f401e89e3648f > # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 > Added dark mode support to error pages - request in ticket #2434 > > diff -r 07b0bee87f32 -r 0dcba2103876 src/http/ngx_http_special_response.c > --- a/src/http/ngx_http_special_response.c Wed Dec 21 14:53:27 2022 +0300 > +++ b/src/http/ngx_http_special_response.c Sat Jan 28 22:37:16 2023 +0000 > @@ -18,6 +18,27 @@ > static ngx_int_t ngx_http_send_refresh(ngx_http_request_t *r); > +#define ngx_error_page_head(MESSAGE) \ > +"" CRLF \ > +"" CRLF \ > +"" MESSAGE "" CRLF \ > +"" CRLF \ > +"" CRLF \ > +"" CRLF \ > +"

" MESSAGE "

" CRLF > + > + > +#define ngx_error_page_head_br(MESSAGE) \ > +"" CRLF \ > +"" CRLF \ > +"400" MESSAGE "" CRLF \ > +"" CRLF \ > +"" CRLF \ > +"" CRLF \ > +"

400 Bad Request

" CRLF \ > +"
" MESSAGE "
" CRLF > + > + > static u_char ngx_http_error_full_tail[] = > "
" NGINX_VER "
" CRLF > "" CRLF > @@ -57,284 +78,140 @@ > "\">" CRLF; > -static char ngx_http_error_301_page[] = > -"" CRLF > -"301 Moved Permanently" CRLF > -"" CRLF > -"

301 Moved Permanently

" CRLF > -; > +static char ngx_http_error_301_page[] = + ngx_error_page_head("301 > Moved Permanently"); > static char ngx_http_error_302_page[] = > -"" CRLF > -"302 Found" CRLF > -"" CRLF > -"

302 Found

" CRLF > -; > + ngx_error_page_head("302 Found"); > static char ngx_http_error_303_page[] = > -"" CRLF > -"303 See Other" CRLF > -"" CRLF > -"

303 See Other

" CRLF > -; > + ngx_error_page_head("303 See Other"); > static char ngx_http_error_307_page[] = > -"" CRLF > -"307 Temporary Redirect" CRLF > -"" CRLF > -"

307 Temporary Redirect

" CRLF > -; > + ngx_error_page_head("307 Temporary Redirect"); > static char ngx_http_error_308_page[] = > -"" CRLF > -"308 Permanent Redirect" CRLF > -"" CRLF > -"

308 Permanent Redirect

" CRLF > -; > + ngx_error_page_head("308 Permanent Redirect"); > static char ngx_http_error_400_page[] = > -"" CRLF > -"400 Bad Request" CRLF > -"" CRLF > -"

400 Bad Request

" CRLF > -; > + ngx_error_page_head("400 Bad Request"); > static char ngx_http_error_401_page[] = > -"" CRLF > -"401 Authorization Required" CRLF > -"" CRLF > -"

401 Authorization Required

" CRLF > -; > + ngx_error_page_head("401 Authorization Required"); > static char ngx_http_error_402_page[] = > -"" CRLF > -"402 Payment Required" CRLF > -"" CRLF > -"

402 Payment Required

" CRLF > -; > + ngx_error_page_head("402 Payment Required"); > static char ngx_http_error_403_page[] = > -"" CRLF > -"403 Forbidden" CRLF > -"" CRLF > -"

403 Forbidden

" CRLF > -; > + ngx_error_page_head("403 Forbidden"); > static char ngx_http_error_404_page[] = > -"" CRLF > -"404 Not Found" CRLF > -"" CRLF > -"

404 Not Found

" CRLF > -; > + ngx_error_page_head("404 Not Found"); > static char ngx_http_error_405_page[] = > -"" CRLF > -"405 Not Allowed" CRLF > -"" CRLF > -"

405 Not Allowed

" CRLF > -; > + ngx_error_page_head("405 Not Allowed"); > static char ngx_http_error_406_page[] = > -"" CRLF > -"406 Not Acceptable" CRLF > -"" CRLF > -"

406 Not Acceptable

" CRLF > -; > + ngx_error_page_head("406 Not Acceptable"); > static char ngx_http_error_408_page[] = > -"" CRLF > -"408 Request Time-out" CRLF > -"" CRLF > -"

408 Request Time-out

" CRLF > -; > + ngx_error_page_head("408 Request Time-out"); > static char ngx_http_error_409_page[] = > -"" CRLF > -"409 Conflict" CRLF > -"" CRLF > -"

409 Conflict

" CRLF > -; > + ngx_error_page_head("409 Conflict"); > static char ngx_http_error_410_page[] = > -"" CRLF > -"410 Gone" CRLF > -"" CRLF > -"

410 Gone

" CRLF > -; > + ngx_error_page_head("410 Gone"); > static char ngx_http_error_411_page[] = > -"" CRLF > -"411 Length Required" CRLF > -"" CRLF > -"

411 Length Required

" CRLF > -; > + ngx_error_page_head("411 Length Required"); > static char ngx_http_error_412_page[] = > -"" CRLF > -"412 Precondition Failed" CRLF > -"" CRLF > -"

412 Precondition Failed

" CRLF > -; > + ngx_error_page_head("412 Precondition Failed"); > static char ngx_http_error_413_page[] = > -"" CRLF > -"413 Request Entity Too Large" CRLF > -"" CRLF > -"

413 Request Entity Too Large

" CRLF > -; > + ngx_error_page_head("413 Request Entity Too Large"); > static char ngx_http_error_414_page[] = > -"" CRLF > -"414 Request-URI Too Large" CRLF > -"" CRLF > -"

414 Request-URI Too Large

" CRLF > -; > + ngx_error_page_head("414 Request-URI Too Large"); > static char ngx_http_error_415_page[] = > -"" CRLF > -"415 Unsupported Media Type" CRLF > -"" CRLF > -"

415 Unsupported Media Type

" CRLF > -; > + ngx_error_page_head("415 Unsupported Media Type"); > static char ngx_http_error_416_page[] = > -"" CRLF > -"416 Requested Range Not Satisfiable" CRLF > -"" CRLF > -"

416 Requested Range Not Satisfiable

" CRLF > -; > + ngx_error_page_head("416 Requested Range Not Satisfiable"); > static char ngx_http_error_421_page[] = > -"" CRLF > -"421 Misdirected Request" CRLF > -"" CRLF > -"

421 Misdirected Request

" CRLF > -; > + ngx_error_page_head("421 Misdirected Request"); > static char ngx_http_error_429_page[] = > -"" CRLF > -"429 Too Many Requests" CRLF > -"" CRLF > -"

429 Too Many Requests

" CRLF > -; > + ngx_error_page_head("429 Too Many Requests"); > -static char ngx_http_error_494_page[] = > -"" CRLF > -"400 Request Header Or Cookie Too Large" > -CRLF > -"" CRLF > -"

400 Bad Request

" CRLF > -"
Request Header Or Cookie Too Large
" CRLF > -; > +static char ngx_http_error_494_page[] = + > ngx_error_page_head_br("Request Header Or Cookie Too Large"); > -static char ngx_http_error_495_page[] = > -"" CRLF > -"400 The SSL certificate error" > -CRLF > -"" CRLF > -"

400 Bad Request

" CRLF > -"
The SSL certificate error
" CRLF > -; > +static char ngx_http_error_495_page[] = + ngx_error_page_head_br("The > SSL certificate error"); static char ngx_http_error_496_page[] = > -"" CRLF > -"400 No required SSL certificate was sent" > -CRLF > -"" CRLF > -"

400 Bad Request

" CRLF > -"
No required SSL certificate was sent
" CRLF > -; > + ngx_error_page_head_br("No required SSL certificate was sent"); > static char ngx_http_error_497_page[] = > -"" CRLF > -"400 The plain HTTP request was sent to HTTPS > port" > -CRLF > -"" CRLF > -"

400 Bad Request

" CRLF > -"
The plain HTTP request was sent to HTTPS port
" CRLF > -; > + ngx_error_page_head_br("The plain HTTP request was sent to HTTPS > port"); static char ngx_http_error_500_page[] = > -"" CRLF > -"500 Internal Server Error" CRLF > -"" CRLF > -"

500 Internal Server Error

" CRLF > -; > + ngx_error_page_head("500 Internal Server Error"); > static char ngx_http_error_501_page[] = > -"" CRLF > -"501 Not Implemented" CRLF > -"" CRLF > -"

501 Not Implemented

" CRLF > -; > + ngx_error_page_head("501 Not Implemented"); > static char ngx_http_error_502_page[] = > -"" CRLF > -"502 Bad Gateway" CRLF > -"" CRLF > -"

502 Bad Gateway

" CRLF > -; > + ngx_error_page_head("502 Bad Gateway"); > static char ngx_http_error_503_page[] = > -"" CRLF > -"503 Service Temporarily Unavailable" CRLF > -"" CRLF > -"

503 Service Temporarily Unavailable

" CRLF > -; > + ngx_error_page_head("503 Service Temporarily Unavailabley"); > static char ngx_http_error_504_page[] = > -"" CRLF > -"504 Gateway Time-out" CRLF > -"" CRLF > -"

504 Gateway Time-out

" CRLF > -; > + ngx_error_page_head("504 Gateway Time-out"); > static char ngx_http_error_505_page[] = > -"" CRLF > -"505 HTTP Version Not Supported" CRLF > -"" CRLF > -"

505 HTTP Version Not Supported

" CRLF > -; > + ngx_error_page_head("505 HTTP Version Not Supported"); > static char ngx_http_error_507_page[] = > -"" CRLF > -"507 Insufficient Storage" CRLF > -"" CRLF > -"

507 Insufficient Storage

" CRLF > -; > + ngx_error_page_head("507 Insufficient Storage"); > static ngx_str_t ngx_http_error_pages[] = { > > > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel -------------- next part -------------- # HG changeset patch # User jordanc.carter at outlook.com # Date 1674945436 0 # Sat Jan 28 22:37:16 2023 +0000 # Branch dark-error # Node ID 0dcba21038765f6f03098cbdf23f401e89e3648f # Parent 07b0bee87f32be91a33210bc06973e07c4c1dac9 Added dark mode support to error pages - request in ticket #2434 diff -r 07b0bee87f32 -r 0dcba2103876 src/http/ngx_http_special_response.c --- a/src/http/ngx_http_special_response.c Wed Dec 21 14:53:27 2022 +0300 +++ b/src/http/ngx_http_special_response.c Sat Jan 28 22:37:16 2023 +0000 @@ -18,6 +18,27 @@ static ngx_int_t ngx_http_send_refresh(ngx_http_request_t *r); +#define ngx_error_page_head(MESSAGE) \ +"" CRLF \ +"" CRLF \ +"" MESSAGE "" CRLF \ +"" CRLF \ +"" CRLF \ +"" CRLF \ +"

" MESSAGE "

" CRLF + + +#define ngx_error_page_head_br(MESSAGE) \ +"" CRLF \ +"" CRLF \ +"400" MESSAGE "" CRLF \ +"" CRLF \ +"" CRLF \ +"" CRLF \ +"

400 Bad Request

" CRLF \ +"
" MESSAGE "
" CRLF + + static u_char ngx_http_error_full_tail[] = "
" NGINX_VER "
" CRLF "" CRLF @@ -57,284 +78,140 @@ "\">" CRLF; -static char ngx_http_error_301_page[] = -"" CRLF -"301 Moved Permanently" CRLF -"" CRLF -"

301 Moved Permanently

" CRLF -; +static char ngx_http_error_301_page[] = + ngx_error_page_head("301 Moved Permanently"); static char ngx_http_error_302_page[] = -"" CRLF -"302 Found" CRLF -"" CRLF -"

302 Found

" CRLF -; + ngx_error_page_head("302 Found"); static char ngx_http_error_303_page[] = -"" CRLF -"303 See Other" CRLF -"" CRLF -"

303 See Other

" CRLF -; + ngx_error_page_head("303 See Other"); static char ngx_http_error_307_page[] = -"" CRLF -"307 Temporary Redirect" CRLF -"" CRLF -"

307 Temporary Redirect

" CRLF -; + ngx_error_page_head("307 Temporary Redirect"); static char ngx_http_error_308_page[] = -"" CRLF -"308 Permanent Redirect" CRLF -"" CRLF -"

308 Permanent Redirect

" CRLF -; + ngx_error_page_head("308 Permanent Redirect"); static char ngx_http_error_400_page[] = -"" CRLF -"400 Bad Request" CRLF -"" CRLF -"

400 Bad Request

" CRLF -; + ngx_error_page_head("400 Bad Request"); static char ngx_http_error_401_page[] = -"" CRLF -"401 Authorization Required" CRLF -"" CRLF -"

401 Authorization Required

" CRLF -; + ngx_error_page_head("401 Authorization Required"); static char ngx_http_error_402_page[] = -"" CRLF -"402 Payment Required" CRLF -"" CRLF -"

402 Payment Required

" CRLF -; + ngx_error_page_head("402 Payment Required"); static char ngx_http_error_403_page[] = -"" CRLF -"403 Forbidden" CRLF -"" CRLF -"

403 Forbidden

" CRLF -; + ngx_error_page_head("403 Forbidden"); static char ngx_http_error_404_page[] = -"" CRLF -"404 Not Found" CRLF -"" CRLF -"

404 Not Found

" CRLF -; + ngx_error_page_head("404 Not Found"); static char ngx_http_error_405_page[] = -"" CRLF -"405 Not Allowed" CRLF -"" CRLF -"

405 Not Allowed

" CRLF -; + ngx_error_page_head("405 Not Allowed"); static char ngx_http_error_406_page[] = -"" CRLF -"406 Not Acceptable" CRLF -"" CRLF -"

406 Not Acceptable

" CRLF -; + ngx_error_page_head("406 Not Acceptable"); static char ngx_http_error_408_page[] = -"" CRLF -"408 Request Time-out" CRLF -"" CRLF -"

408 Request Time-out

" CRLF -; + ngx_error_page_head("408 Request Time-out"); static char ngx_http_error_409_page[] = -"" CRLF -"409 Conflict" CRLF -"" CRLF -"

409 Conflict

" CRLF -; + ngx_error_page_head("409 Conflict"); static char ngx_http_error_410_page[] = -"" CRLF -"410 Gone" CRLF -"" CRLF -"

410 Gone

" CRLF -; + ngx_error_page_head("410 Gone"); static char ngx_http_error_411_page[] = -"" CRLF -"411 Length Required" CRLF -"" CRLF -"

411 Length Required

" CRLF -; + ngx_error_page_head("411 Length Required"); static char ngx_http_error_412_page[] = -"" CRLF -"412 Precondition Failed" CRLF -"" CRLF -"

412 Precondition Failed

" CRLF -; + ngx_error_page_head("412 Precondition Failed"); static char ngx_http_error_413_page[] = -"" CRLF -"413 Request Entity Too Large" CRLF -"" CRLF -"

413 Request Entity Too Large

" CRLF -; + ngx_error_page_head("413 Request Entity Too Large"); static char ngx_http_error_414_page[] = -"" CRLF -"414 Request-URI Too Large" CRLF -"" CRLF -"

414 Request-URI Too Large

" CRLF -; + ngx_error_page_head("414 Request-URI Too Large"); static char ngx_http_error_415_page[] = -"" CRLF -"415 Unsupported Media Type" CRLF -"" CRLF -"

415 Unsupported Media Type

" CRLF -; + ngx_error_page_head("415 Unsupported Media Type"); static char ngx_http_error_416_page[] = -"" CRLF -"416 Requested Range Not Satisfiable" CRLF -"" CRLF -"

416 Requested Range Not Satisfiable

" CRLF -; + ngx_error_page_head("416 Requested Range Not Satisfiable"); static char ngx_http_error_421_page[] = -"" CRLF -"421 Misdirected Request" CRLF -"" CRLF -"

421 Misdirected Request

" CRLF -; + ngx_error_page_head("421 Misdirected Request"); static char ngx_http_error_429_page[] = -"" CRLF -"429 Too Many Requests" CRLF -"" CRLF -"

429 Too Many Requests

" CRLF -; + ngx_error_page_head("429 Too Many Requests"); -static char ngx_http_error_494_page[] = -"" CRLF -"400 Request Header Or Cookie Too Large" -CRLF -"" CRLF -"

400 Bad Request

" CRLF -"
Request Header Or Cookie Too Large
" CRLF -; +static char ngx_http_error_494_page[] = + ngx_error_page_head_br("Request Header Or Cookie Too Large"); -static char ngx_http_error_495_page[] = -"" CRLF -"400 The SSL certificate error" -CRLF -"" CRLF -"

400 Bad Request

" CRLF -"
The SSL certificate error
" CRLF -; +static char ngx_http_error_495_page[] = + ngx_error_page_head_br("The SSL certificate error"); static char ngx_http_error_496_page[] = -"" CRLF -"400 No required SSL certificate was sent" -CRLF -"" CRLF -"

400 Bad Request

" CRLF -"
No required SSL certificate was sent
" CRLF -; + ngx_error_page_head_br("No required SSL certificate was sent"); static char ngx_http_error_497_page[] = -"" CRLF -"400 The plain HTTP request was sent to HTTPS port" -CRLF -"" CRLF -"

400 Bad Request

" CRLF -"
The plain HTTP request was sent to HTTPS port
" CRLF -; + ngx_error_page_head_br("The plain HTTP request was sent to HTTPS port"); static char ngx_http_error_500_page[] = -"" CRLF -"500 Internal Server Error" CRLF -"" CRLF -"

500 Internal Server Error

" CRLF -; + ngx_error_page_head("500 Internal Server Error"); static char ngx_http_error_501_page[] = -"" CRLF -"501 Not Implemented" CRLF -"" CRLF -"

501 Not Implemented

" CRLF -; + ngx_error_page_head("501 Not Implemented"); static char ngx_http_error_502_page[] = -"" CRLF -"502 Bad Gateway" CRLF -"" CRLF -"

502 Bad Gateway

" CRLF -; + ngx_error_page_head("502 Bad Gateway"); static char ngx_http_error_503_page[] = -"" CRLF -"503 Service Temporarily Unavailable" CRLF -"" CRLF -"

503 Service Temporarily Unavailable

" CRLF -; + ngx_error_page_head("503 Service Temporarily Unavailabley"); static char ngx_http_error_504_page[] = -"" CRLF -"504 Gateway Time-out" CRLF -"" CRLF -"

504 Gateway Time-out

" CRLF -; + ngx_error_page_head("504 Gateway Time-out"); static char ngx_http_error_505_page[] = -"" CRLF -"505 HTTP Version Not Supported" CRLF -"" CRLF -"

505 HTTP Version Not Supported

" CRLF -; + ngx_error_page_head("505 HTTP Version Not Supported"); static char ngx_http_error_507_page[] = -"" CRLF -"507 Insufficient Storage" CRLF -"" CRLF -"

507 Insufficient Storage

" CRLF -; + ngx_error_page_head("507 Insufficient Storage"); static ngx_str_t ngx_http_error_pages[] = { From mdounin at mdounin.ru Mon Jan 30 01:25:51 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 30 Jan 2023 04:25:51 +0300 Subject: [PATCH 1 of 3] HTTP/3: "quic" parameter of "listen" directive In-Reply-To: <250539ea3e05d0ac2ead.1674733845@arut-laptop> References: <250539ea3e05d0ac2ead.1674733845@arut-laptop> Message-ID: Hello! On Thu, Jan 26, 2023 at 03:50:45PM +0400, Roman Arutyunyan wrote: > # HG changeset patch > # User Roman Arutyunyan > # Date 1674653899 -14400 > # Wed Jan 25 17:38:19 2023 +0400 > # Branch quic > # Node ID 250539ea3e05d0ac2ead4784a1b44f28ff4d1ef6 > # Parent a954b551dc3f8f059140e5843c49341cae55b98b > HTTP/3: "quic" parameter of "listen" directive. > > Now "listen" directve has a new "quic" parameter which enables QUIC protocol > for the address. Further, to enable HTTP/3, a new directive "http3" is > introduced. The hq-interop protocol is enabled by "http3_hq" as before. > Now application protocol is chosen by ALPN. > > Previously used "http3" parameter of "listen" is eliminated. In no particular order: - As previously discussed, the "quic" option still implies "udp", which is essentially a part of address. Not sure what to do with this. - It looks like it would be possible to use non-QUIC name-based virtual servers over QUIC. Should we disable it (and return 421)? - Some compatibility shims for previous configurations (not to be merged into mainline, but might be useful for nginx-quic for a while)? [...] -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Mon Jan 30 01:26:27 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 30 Jan 2023 04:26:27 +0300 Subject: [PATCH 2 of 3] HTTP/3: trigger more compatibility errors for "listen quic" In-Reply-To: <555913c358221f647bba.1674733846@arut-laptop> References: <555913c358221f647bba.1674733846@arut-laptop> Message-ID: Hello! On Thu, Jan 26, 2023 at 03:50:46PM +0400, Roman Arutyunyan wrote: > # HG changeset patch > # User Roman Arutyunyan > # Date 1674732333 -14400 > # Thu Jan 26 15:25:33 2023 +0400 > # Branch quic > # Node ID 555913c358221f647bbace26165bef5eb614add4 > # Parent 250539ea3e05d0ac2ead4784a1b44f28ff4d1ef6 > HTTP/3: trigger more compatibility errors for "listen quic". > > Now "ssl", "proxy_protocol" and "http2" are not allowed with "quic" in "listen" > directive. Previously, only "ssl" was not allowed. > > diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c > --- a/src/http/ngx_http_core_module.c > +++ b/src/http/ngx_http_core_module.c > @@ -4303,10 +4303,26 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx > return NGX_CONF_ERROR; > } > > -#if (NGX_HTTP_SSL && NGX_HTTP_V3) > - if (lsopt.ssl && lsopt.quic) { > - return "\"ssl\" parameter is incompatible with \"quic\""; > - } > +#if (NGX_HTTP_V3) > + > + if (lsopt.quic) { > +#if (NGX_HTTP_SSL) > + if (lsopt.ssl) { > + return "\"ssl\" parameter is incompatible with \"quic\""; > + } > +#endif > + > +#if (NGX_HTTP_V2) > + if (lsopt.http2) { > + return "\"http2\" parameter is incompatible with \"quic\""; > + } > +#endif > + > + if (lsopt.proxy_protocol) { > + return "\"proxy_protocol\" parameter is incompatible with \"quic\""; > + } > + } > + > #endif > > for (n = 0; n < u.naddrs; n++) { Looks good. -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Mon Jan 30 01:29:00 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 30 Jan 2023 04:29:00 +0300 Subject: [PATCH 3 of 3] HTTP/2: "http2" directive In-Reply-To: <819737783463d7e38ea8.1674733847@arut-laptop> References: <819737783463d7e38ea8.1674733847@arut-laptop> Message-ID: Hello! On Thu, Jan 26, 2023 at 03:50:47PM +0400, Roman Arutyunyan wrote: > # HG changeset patch > # User Roman Arutyunyan > # Date 1674649725 -14400 > # Wed Jan 25 16:28:45 2023 +0400 > # Branch quic > # Node ID 819737783463d7e38ea80109a976db1d3a9bb2db > # Parent 555913c358221f647bbace26165bef5eb614add4 > HTTP/2: "http2" directive. > > The directive enables HTTP/2 in the current server. The previous way to > enable HTTP/2 via "listen ... http2" is now deprecated. In no particular order: - The ngx_http_try_v2() should be a separate patch? - The "(buf[0] == 'P')" check is not going to work, since valid HTTP/1.x request can start with 'P' ('POST', 'PATCH', et cetera). This needs better checking. Further, it's not guaranteed to arrive in a single packet. - Virtual servers (and 421), see comments to the first patch. -- Maxim Dounin http://mdounin.ru/ From alx.manpages at gmail.com Mon Jan 30 13:35:53 2023 From: alx.manpages at gmail.com (Alejandro Colomar) Date: Mon, 30 Jan 2023 14:35:53 +0100 Subject: [PATCH 1/2] Added NJS_MALLOC(). Message-ID: <20230130133553.8477-1-alx@nginx.com> This attribute helps avoid memory leaks. Link: Cc: Liam Crilly Cc: Zhidao Hong Signed-off-by: Alejandro Colomar --- src/njs_clang.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/njs_clang.h b/src/njs_clang.h index 614b509..c52d75a 100644 --- a/src/njs_clang.h +++ b/src/njs_clang.h @@ -155,10 +155,12 @@ njs_leading_zeros64(uint64_t x) #if (NJS_HAVE_GCC_ATTRIBUTE_MALLOC) -#define NJS_MALLOC_LIKE __attribute__((__malloc__)) +#define NJS_MALLOC_LIKE __attribute__((__malloc__)) +#define NJS_MALLOC(deallocator) __attribute__((__malloc__(deallocator))) #else #define NJS_MALLOC_LIKE +#define NJS_MALLOC(deallocator) #endif -- 2.39.0 From alx.manpages at gmail.com Mon Jan 30 13:35:55 2023 From: alx.manpages at gmail.com (Alejandro Colomar) Date: Mon, 30 Jan 2023 14:35:55 +0100 Subject: [PATCH 2/2] Using NJS_MALLOC() in functions that create new objects. In-Reply-To: <20230130133553.8477-1-alx@nginx.com> References: <20230130133553.8477-1-alx@nginx.com> Message-ID: <20230130133553.8477-2-alx@nginx.com> NGINX Unit introduced a memory leak recently due to forgetting to call the destructor. This could probably have been warned at compile time, or maybe at link time, by the compiler, if we had used the GNU attribute. Let's use it to prevent misuse of the APIs. It also helps self-document through source code how they should be used properly. The bug in Unit was found by Liam Crilly. Cc: Liam Crilly Cc: Zhidao Hong Signed-off-by: Alejandro Colomar --- src/njs.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/njs.h b/src/njs.h index 95c4125..86b5a8a 100644 --- a/src/njs.h +++ b/src/njs.h @@ -306,15 +306,17 @@ typedef struct { NJS_EXPORT void njs_vm_opt_init(njs_vm_opt_t *options); -NJS_EXPORT njs_vm_t *njs_vm_create(njs_vm_opt_t *options); NJS_EXPORT void njs_vm_destroy(njs_vm_t *vm); +NJS_EXPORT NJS_MALLOC(njs_vm_destroy) njs_vm_t *njs_vm_create( + njs_vm_opt_t *options); NJS_EXPORT njs_int_t njs_vm_compile(njs_vm_t *vm, u_char **start, u_char *end); NJS_EXPORT njs_mod_t *njs_vm_add_module(njs_vm_t *vm, njs_str_t *name, njs_value_t *value); NJS_EXPORT njs_mod_t *njs_vm_compile_module(njs_vm_t *vm, njs_str_t *name, u_char **start, u_char *end); -NJS_EXPORT njs_vm_t *njs_vm_clone(njs_vm_t *vm, njs_external_ptr_t external); +NJS_EXPORT NJS_MALLOC(njs_vm_destroy) njs_vm_t *njs_vm_clone(njs_vm_t *vm, + njs_external_ptr_t external); NJS_EXPORT njs_vm_event_t njs_vm_add_event(njs_vm_t *vm, njs_function_t *function, njs_uint_t once, njs_host_event_t host_ev, -- 2.39.0 From pluknet at nginx.com Mon Jan 30 14:57:15 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Mon, 30 Jan 2023 14:57:15 +0000 Subject: [nginx] Fixed handling of very long locations (ticket #2435). Message-ID: details: https://hg.nginx.org/nginx/rev/4eb1383f6432 branches: changeset: 8121:4eb1383f6432 user: Maxim Dounin date: Thu Jan 26 03:34:44 2023 +0300 description: Fixed handling of very long locations (ticket #2435). Previously, location prefix length in ngx_http_location_tree_node_t was stored as "u_char", and therefore location prefixes longer than 255 bytes were handled incorrectly. Fix is to use "u_short" instead. With "u_short", prefixes up to 65535 bytes can be safely used, and this isn't reachable due to NGX_CONF_BUFFER, which is 4096 bytes. diffstat: src/http/ngx_http.c | 2 +- src/http/ngx_http_core_module.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (25 lines): diff -r c7e103acb409 -r 4eb1383f6432 src/http/ngx_http.c --- a/src/http/ngx_http.c Tue Jan 24 03:01:51 2023 +0300 +++ b/src/http/ngx_http.c Thu Jan 26 03:34:44 2023 +0300 @@ -1130,7 +1130,7 @@ ngx_http_create_locations_tree(ngx_conf_ node->auto_redirect = (u_char) ((lq->exact && lq->exact->auto_redirect) || (lq->inclusive && lq->inclusive->auto_redirect)); - node->len = (u_char) len; + node->len = (u_short) len; ngx_memcpy(node->name, &lq->name->data[prefix], len); ngx_queue_split(locations, q, &tail); diff -r c7e103acb409 -r 4eb1383f6432 src/http/ngx_http_core_module.h --- a/src/http/ngx_http_core_module.h Tue Jan 24 03:01:51 2023 +0300 +++ b/src/http/ngx_http_core_module.h Thu Jan 26 03:34:44 2023 +0300 @@ -463,8 +463,8 @@ struct ngx_http_location_tree_node_s { ngx_http_core_loc_conf_t *exact; ngx_http_core_loc_conf_t *inclusive; + u_short len; u_char auto_redirect; - u_char len; u_char name[1]; }; From pluknet at nginx.com Mon Jan 30 14:57:18 2023 From: pluknet at nginx.com (Sergey Kandaurov) Date: Mon, 30 Jan 2023 14:57:18 +0000 Subject: [nginx] Added warning about redefinition of listen socket protocol options. Message-ID: details: https://hg.nginx.org/nginx/rev/106328a70f4e branches: changeset: 8122:106328a70f4e user: Maxim Dounin date: Sat Jan 28 01:29:45 2023 +0300 description: Added warning about redefinition of listen socket protocol options. The "listen" directive in the http module can be used multiple times in different server blocks. Originally, it was supposed to be specified once with various socket options, and without any parameters in virtual server blocks. For example: server { listen 80 backlog=1024; server_name foo; ... } server { listen 80; server_name bar; ... } server { listen 80; server_name bazz; ... } The address part of the syntax ("address[:port]" / "port" / "unix:path") uniquely identifies the listening socket, and therefore is enough for name-based virtual servers (to let nginx know that the virtual server accepts requests on the listening socket in question). To ensure that listening options do not conflict between virtual servers, they were allowed only once. For example, the following configuration will be rejected ("duplicate listen options for 0.0.0.0:80 in ..."): server { listen 80 backlog=1024; server_name foo; ... } server { listen 80 backlog=512; server_name bar; ... } At some point it was, however, noticed, that it is sometimes convenient to repeat some options for clarity. In nginx 0.8.51 the "ssl" parameter was allowed to be specified multiple times, e.g.: server { listen 443 ssl backlog=1024; server_name foo; ... } server { listen 443 ssl; server_name bar; ... } server { listen 443 ssl; server_name bazz; ... } This approach makes configuration more readable, since SSL sockets are immediately visible in the configuration. If this is not needed, just the address can still be used. Later, additional protocol-specific options similar to "ssl" were introduced, notably "http2" and "proxy_protocol". With these options, one can write: server { listen 443 ssl backlog=1024; server_name foo; ... } server { listen 443 http2; server_name bar; ... } server { listen 443 proxy_protocol; server_name bazz; ... } The resulting socket will use ssl, http2, and proxy_protocol, but this is not really obvious from the configuration. To emphasize such misleading configurations are discouraged, nginx now warns as long as the "listen" directive is used with options different from the options previously used if this is potentially confusing. In particular, the following configurations are allowed: server { listen 8401 ssl backlog=1024; server_name foo; } server { listen 8401 ssl; server_name bar; } server { listen 8401 ssl; server_name bazz; } server { listen 8402 ssl http2 backlog=1024; server_name foo; } server { listen 8402 ssl; server_name bar; } server { listen 8402 ssl; server_name bazz; } server { listen 8403 ssl; server_name bar; } server { listen 8403 ssl; server_name bazz; } server { listen 8403 ssl http2; server_name foo; } server { listen 8404 ssl http2 backlog=1024; server_name foo; } server { listen 8404 http2; server_name bar; } server { listen 8404 http2; server_name bazz; } server { listen 8405 ssl http2 backlog=1024; server_name foo; } server { listen 8405 ssl http2; server_name bar; } server { listen 8405 ssl http2; server_name bazz; } server { listen 8406 ssl; server_name foo; } server { listen 8406; server_name bar; } server { listen 8406; server_name bazz; } And the following configurations will generate warnings: server { listen 8501 ssl http2 backlog=1024; server_name foo; } server { listen 8501 http2; server_name bar; } server { listen 8501 ssl; server_name bazz; } server { listen 8502 backlog=1024; server_name foo; } server { listen 8502 ssl; server_name bar; } server { listen 8503 ssl; server_name foo; } server { listen 8503 http2; server_name bar; } server { listen 8504 ssl; server_name foo; } server { listen 8504 http2; server_name bar; } server { listen 8504 proxy_protocol; server_name bazz; } server { listen 8505 ssl http2 proxy_protocol; server_name foo; } server { listen 8505 ssl http2; server_name bar; } server { listen 8505 ssl; server_name bazz; } server { listen 8506 ssl http2; server_name foo; } server { listen 8506 ssl; server_name bar; } server { listen 8506; server_name bazz; } server { listen 8507 ssl; server_name bar; } server { listen 8507; server_name bazz; } server { listen 8507 ssl http2; server_name foo; } server { listen 8508 ssl; server_name bar; } server { listen 8508; server_name bazz; } server { listen 8508 ssl backlog=1024; server_name foo; } server { listen 8509; server_name bazz; } server { listen 8509 ssl; server_name bar; } server { listen 8509 ssl backlog=1024; server_name foo; } The basic idea is that at most two sets of protocol options are allowed: the main one (with socket options, if any), and a shorter one, with options being a subset of the main options, repeated for clarity. As long as the shorter set of protocol options is used, all listen directives except the main one should use it. diffstat: src/http/ngx_http.c | 63 ++++++++++++++++++++++++++++++++++++++++- src/http/ngx_http_core_module.h | 4 ++ 2 files changed, 66 insertions(+), 1 deletions(-) diffs (114 lines): diff -r 4eb1383f6432 -r 106328a70f4e src/http/ngx_http.c --- a/src/http/ngx_http.c Thu Jan 26 03:34:44 2023 +0300 +++ b/src/http/ngx_http.c Sat Jan 28 01:29:45 2023 +0300 @@ -1228,7 +1228,8 @@ static ngx_int_t ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, ngx_http_conf_port_t *port, ngx_http_listen_opt_t *lsopt) { - ngx_uint_t i, default_server, proxy_protocol; + ngx_uint_t i, default_server, proxy_protocol, + protocols, protocols_prev; ngx_http_conf_addr_t *addr; #if (NGX_HTTP_SSL) ngx_uint_t ssl; @@ -1264,12 +1265,18 @@ ngx_http_add_addresses(ngx_conf_t *cf, n default_server = addr[i].opt.default_server; proxy_protocol = lsopt->proxy_protocol || addr[i].opt.proxy_protocol; + protocols = lsopt->proxy_protocol; + protocols_prev = addr[i].opt.proxy_protocol; #if (NGX_HTTP_SSL) ssl = lsopt->ssl || addr[i].opt.ssl; + protocols |= lsopt->ssl << 1; + protocols_prev |= addr[i].opt.ssl << 1; #endif #if (NGX_HTTP_V2) http2 = lsopt->http2 || addr[i].opt.http2; + protocols |= lsopt->http2 << 2; + protocols_prev |= addr[i].opt.http2 << 2; #endif if (lsopt->set) { @@ -1299,6 +1306,57 @@ ngx_http_add_addresses(ngx_conf_t *cf, n addr[i].default_server = cscf; } + /* check for conflicting protocol options */ + + if ((protocols | protocols_prev) != protocols_prev) { + + /* options added */ + + if ((addr[i].opt.set && !lsopt->set) + || addr[i].protocols_changed + || (protocols | protocols_prev) != protocols) + { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "protocol options redefined for %V", + &addr[i].opt.addr_text); + } + + addr[i].protocols = protocols_prev; + addr[i].protocols_set = 1; + addr[i].protocols_changed = 1; + + } else if ((protocols_prev | protocols) != protocols) { + + /* options removed */ + + if (lsopt->set + || (addr[i].protocols_set && protocols != addr[i].protocols)) + { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "protocol options redefined for %V", + &addr[i].opt.addr_text); + } + + addr[i].protocols = protocols; + addr[i].protocols_set = 1; + addr[i].protocols_changed = 1; + + } else { + + /* the same options */ + + if ((lsopt->set && addr[i].protocols_changed) + || (addr[i].protocols_set && protocols != addr[i].protocols)) + { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "protocol options redefined for %V", + &addr[i].opt.addr_text); + } + + addr[i].protocols = protocols; + addr[i].protocols_set = 1; + } + addr[i].opt.default_server = default_server; addr[i].opt.proxy_protocol = proxy_protocol; #if (NGX_HTTP_SSL) @@ -1355,6 +1413,9 @@ ngx_http_add_address(ngx_conf_t *cf, ngx } addr->opt = *lsopt; + addr->protocols = 0; + addr->protocols_set = 0; + addr->protocols_changed = 0; addr->hash.buckets = NULL; addr->hash.size = 0; addr->wc_head = NULL; diff -r 4eb1383f6432 -r 106328a70f4e src/http/ngx_http_core_module.h --- a/src/http/ngx_http_core_module.h Thu Jan 26 03:34:44 2023 +0300 +++ b/src/http/ngx_http_core_module.h Sat Jan 28 01:29:45 2023 +0300 @@ -274,6 +274,10 @@ typedef struct { typedef struct { ngx_http_listen_opt_t opt; + unsigned protocols:3; + unsigned protocols_set:1; + unsigned protocols_changed:1; + ngx_hash_t hash; ngx_hash_wildcard_t *wc_head; ngx_hash_wildcard_t *wc_tail; From thresh at nginx.com Tue Jan 31 00:52:42 2023 From: thresh at nginx.com (Konstantin Pavlov) Date: Tue, 31 Jan 2023 00:52:42 +0000 Subject: [njs] Added description for --no-libxml2 option. Message-ID: details: https://hg.nginx.org/njs/rev/b0f3bc578f08 branches: changeset: 2029:b0f3bc578f08 user: Konstantin Pavlov date: Fri Jan 27 15:08:41 2023 -0800 description: Added description for --no-libxml2 option. diffstat: auto/help | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 99b9f83e4d4d -r b0f3bc578f08 auto/help --- a/auto/help Wed Jan 25 21:54:47 2023 -0800 +++ b/auto/help Fri Jan 27 15:08:41 2023 -0800 @@ -35,6 +35,10 @@ default: "$NJS_LD_OPT" enabled OpenSSL dependant code is not built as a part of libnjs.a. + --no-libxml2 disabled libxml2 discovery. When this option is + enabled libxml2 dependant code is not built as a + part of libnjs.a. + --address-sanitizer=YES enables build with address sanitizer, \ default: "$NJS_ADDRESS_SANITIZER" --addr2line=YES enables native function symbolization, \ From thresh at nginx.com Tue Jan 31 00:52:44 2023 From: thresh at nginx.com (Konstantin Pavlov) Date: Tue, 31 Jan 2023 00:52:44 +0000 Subject: [njs] Added pkg-config-based libxml2 detection. Message-ID: details: https://hg.nginx.org/njs/rev/c789bb7313d4 branches: changeset: 2030:c789bb7313d4 user: Konstantin Pavlov date: Fri Jan 27 15:10:57 2023 -0800 description: Added pkg-config-based libxml2 detection. diffstat: auto/libxml2 | 26 +++++++++++++++++++++----- 1 files changed, 21 insertions(+), 5 deletions(-) diffs (45 lines): diff -r b0f3bc578f08 -r c789bb7313d4 auto/libxml2 --- a/auto/libxml2 Fri Jan 27 15:08:41 2023 -0800 +++ b/auto/libxml2 Fri Jan 27 15:10:57 2023 -0800 @@ -6,12 +6,8 @@ NJS_HAVE_LIBXML2=NO if [ $NJS_LIBXML2 = YES ]; then njs_found=no - - njs_feature="libxml2" njs_feature_name=NJS_HAVE_LIBXML2 njs_feature_run=no - njs_feature_incs="/usr/include/libxml2" - njs_feature_libs="-lxml2" njs_feature_test="#include #include @@ -22,7 +18,27 @@ if [ $NJS_LIBXML2 = YES ]; then xmlCleanupParser(); return 0; }" - . auto/feature + + + if /bin/sh -c "(pkg-config libxml-2.0 --exists)" >> $NJS_AUTOCONF_ERR 2>&1; then + + # pkg-config + + njs_feature="libxml2 via pkg-config" + njs_feature_incs=`pkg-config libxml-2.0 --cflags | sed -n -e 's/.*-I *\([^ ][^ ]*\).*/\1/p'` + njs_feature_libs=`pkg-config libxml-2.0 --libs` + + . auto/feature + fi + + if [ $njs_found = no ]; then + + njs_feature="libxml2" + njs_feature_incs="/usr/include/libxml2" + njs_feature_libs="-lxml2" + + . auto/feature + fi if [ $njs_found = no ]; then From thresh at nginx.com Tue Jan 31 00:52:46 2023 From: thresh at nginx.com (Konstantin Pavlov) Date: Tue, 31 Jan 2023 00:52:46 +0000 Subject: [njs] Report libxml2 used for build. Message-ID: details: https://hg.nginx.org/njs/rev/cba6c332ef5a branches: changeset: 2031:cba6c332ef5a user: Konstantin Pavlov date: Fri Jan 27 15:19:38 2023 -0800 description: Report libxml2 used for build. diffstat: auto/libxml2 | 2 ++ auto/summary | 4 ++++ 2 files changed, 6 insertions(+), 0 deletions(-) diffs (33 lines): diff -r c789bb7313d4 -r cba6c332ef5a auto/libxml2 --- a/auto/libxml2 Fri Jan 27 15:10:57 2023 -0800 +++ b/auto/libxml2 Fri Jan 27 15:19:38 2023 -0800 @@ -2,6 +2,7 @@ # Copyright (C) Dmitry Volyntsev # Copyright (C) NGINX, Inc. +NJS_LIBXML2_LIB= NJS_HAVE_LIBXML2=NO if [ $NJS_LIBXML2 = YES ]; then @@ -87,6 +88,7 @@ if [ $NJS_LIBXML2 = YES ]; then . auto/feature NJS_HAVE_LIBXML2=YES + NJS_LIBXML2_LIB="$njs_feature_libs" NJS_LIB_INCS="$NJS_LIB_INCS $njs_feature_incs" NJS_LIB_AUX_LIBS="$NJS_LIB_AUX_LIBS $njs_feature_libs" fi diff -r c789bb7313d4 -r cba6c332ef5a auto/summary --- a/auto/summary Fri Jan 27 15:10:57 2023 -0800 +++ b/auto/summary Fri Jan 27 15:19:38 2023 -0800 @@ -22,6 +22,10 @@ if [ $NJS_HAVE_OPENSSL = YES ]; then echo " + using OpenSSL library: $NJS_OPENSSL_LIB" fi +if [ $NJS_HAVE_LIBXML2 = YES ]; then + echo " + using libxml2 library: $NJS_LIBXML2_LIB" +fi + if [ $NJS_HAVE_COMPUTED_GOTO = YES ]; then echo " + using computed goto" fi From alx.manpages at gmail.com Tue Jan 31 01:52:25 2023 From: alx.manpages at gmail.com (Alejandro Colomar) Date: Tue, 31 Jan 2023 02:52:25 +0100 Subject: [njs] Added pkg-config-based libxml2 detection. In-Reply-To: References: Message-ID: Hi Konstantin! On 1/31/23 01:52, Konstantin Pavlov wrote: > details: https://hg.nginx.org/njs/rev/c789bb7313d4 > branches: > changeset: 2030:c789bb7313d4 > user: Konstantin Pavlov > date: Fri Jan 27 15:10:57 2023 -0800 > description: > Added pkg-config-based libxml2 detection. > > diffstat: > > auto/libxml2 | 26 +++++++++++++++++++++----- > 1 files changed, 21 insertions(+), 5 deletions(-) > > diffs (45 lines): > > diff -r b0f3bc578f08 -r c789bb7313d4 auto/libxml2 > --- a/auto/libxml2 Fri Jan 27 15:08:41 2023 -0800 > +++ b/auto/libxml2 Fri Jan 27 15:10:57 2023 -0800 > @@ -6,12 +6,8 @@ NJS_HAVE_LIBXML2=NO > > if [ $NJS_LIBXML2 = YES ]; then > njs_found=no > - > - njs_feature="libxml2" > njs_feature_name=NJS_HAVE_LIBXML2 > njs_feature_run=no > - njs_feature_incs="/usr/include/libxml2" > - njs_feature_libs="-lxml2" > njs_feature_test="#include > #include > > @@ -22,7 +18,27 @@ if [ $NJS_LIBXML2 = YES ]; then > xmlCleanupParser(); > return 0; > }" > - . auto/feature > + > + > + if /bin/sh -c "(pkg-config libxml-2.0 --exists)" >> $NJS_AUTOCONF_ERR 2>&1; then > + > + # pkg-config > + > + njs_feature="libxml2 via pkg-config" > + njs_feature_incs=`pkg-config libxml-2.0 --cflags | sed -n -e 's/.*-I *\([^ ][^ ]*\).*/\1/p'` I think you want -‐cflags‐only‐I I'm not sure if it has any portability issues to old versions of pkg-config maybe? Cheers, Alex > + njs_feature_libs=`pkg-config libxml-2.0 --libs` > + > + . auto/feature > + fi > + > + if [ $njs_found = no ]; then > + > + njs_feature="libxml2" > + njs_feature_incs="/usr/include/libxml2" > + njs_feature_libs="-lxml2" > + > + . auto/feature > + fi > > if [ $njs_found = no ]; then > > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel -- -------------- next part -------------- A non-text attachment was scrubbed... Name: OpenPGP_signature Type: application/pgp-signature Size: 833 bytes Desc: OpenPGP digital signature URL: From thresh at nginx.com Tue Jan 31 03:24:50 2023 From: thresh at nginx.com (Konstantin Pavlov) Date: Mon, 30 Jan 2023 19:24:50 -0800 Subject: [njs] Added pkg-config-based libxml2 detection. In-Reply-To: References: Message-ID: <21b3469a-6209-54e1-61d8-55e4495ac07a@nginx.com> Hi Alejandro, On 30/01/2023 5:52 PM, Alejandro Colomar wrote: > Hi Konstantin! > > On 1/31/23 01:52, Konstantin Pavlov wrote: >> details: https://hg.nginx.org/njs/rev/c789bb7313d4 >> branches: >> changeset: 2030:c789bb7313d4 >> user:      Konstantin Pavlov >> date:      Fri Jan 27 15:10:57 2023 -0800 >> description: >> Added pkg-config-based libxml2 detection. >> >> diffstat: >> >>   auto/libxml2 |  26 +++++++++++++++++++++----- >>   1 files changed, 21 insertions(+), 5 deletions(-) >> >> diffs (45 lines): >> >> diff -r b0f3bc578f08 -r c789bb7313d4 auto/libxml2 >> --- a/auto/libxml2    Fri Jan 27 15:08:41 2023 -0800 >> +++ b/auto/libxml2    Fri Jan 27 15:10:57 2023 -0800 >> @@ -6,12 +6,8 @@ NJS_HAVE_LIBXML2=NO >>     if [ $NJS_LIBXML2 = YES ]; then >>       njs_found=no >> - >> -    njs_feature="libxml2" >>       njs_feature_name=NJS_HAVE_LIBXML2 >>       njs_feature_run=no >> -    njs_feature_incs="/usr/include/libxml2" >> -    njs_feature_libs="-lxml2" >>       njs_feature_test="#include >>                         #include >>   @@ -22,7 +18,27 @@ if [ $NJS_LIBXML2 = YES ]; then >>                             xmlCleanupParser(); >>                             return 0; >>                         }" >> -    . auto/feature >> + >> + >> +    if /bin/sh -c "(pkg-config libxml-2.0 --exists)" >> >> $NJS_AUTOCONF_ERR 2>&1; then >> + >> +        # pkg-config >> + >> +        njs_feature="libxml2 via pkg-config" >> +        njs_feature_incs=`pkg-config libxml-2.0 --cflags | sed -n -e >> 's/.*-I *\([^ ][^ ]*\).*/\1/p'` > > I think you want -‐cflags‐only‐I > I'm not sure if it has any portability issues to old versions of > pkg-config maybe? Do you mean so pkg-config only outputs -I/foo/bar but not any non-I parts of cflags like -Dfoo if any?  Maybe. Not sure it's a big deal though? From alx.manpages at gmail.com Tue Jan 31 10:40:15 2023 From: alx.manpages at gmail.com (Alejandro Colomar) Date: Tue, 31 Jan 2023 11:40:15 +0100 Subject: [njs] Added pkg-config-based libxml2 detection. In-Reply-To: <21b3469a-6209-54e1-61d8-55e4495ac07a@nginx.com> References: <21b3469a-6209-54e1-61d8-55e4495ac07a@nginx.com> Message-ID: <72c54109-bb50-8963-0a2c-b6ad4c99de8b@gmail.com> Hi Konstantin, On 1/31/23 04:24, Konstantin Pavlov wrote: >>> @@ -6,12 +6,8 @@ NJS_HAVE_LIBXML2=NO >>>     if [ $NJS_LIBXML2 = YES ]; then >>>       njs_found=no >>> - >>> -    njs_feature="libxml2" >>>       njs_feature_name=NJS_HAVE_LIBXML2 >>>       njs_feature_run=no >>> -    njs_feature_incs="/usr/include/libxml2" >>> -    njs_feature_libs="-lxml2" >>>       njs_feature_test="#include >>>                         #include >>>   @@ -22,7 +18,27 @@ if [ $NJS_LIBXML2 = YES ]; then >>>                             xmlCleanupParser(); >>>                             return 0; >>>                         }" >>> -    . auto/feature >>> + >>> + >>> +    if /bin/sh -c "(pkg-config libxml-2.0 --exists)" >> $NJS_AUTOCONF_ERR >>> 2>&1; then >>> + >>> +        # pkg-config >>> + >>> +        njs_feature="libxml2 via pkg-config" >>> +        njs_feature_incs=`pkg-config libxml-2.0 --cflags | sed -n -e 's/.*-I >>> *\([^ ][^ ]*\).*/\1/p'` >> >> I think you want -‐cflags‐only‐I >> I'm not sure if it has any portability issues to old versions of pkg-config >> maybe? > > Do you mean so pkg-config only outputs -I/foo/bar but not any non-I parts of > cflags like -Dfoo if any? Exactly :) From the pkgconf(1) manual page: -‐cflags, -‐cflags‐only‐I, -‐cflags‐only‐other Display either all CFLAGS, only -I CFLAGS or only CFLAGS that are not -I. I checked that the pkg-config_0.29.orig.tar.gz tarball from Debian Stretch, which is as far back as I can go, already provides this flag, so I guess it's reasonable to use it. >  Maybe. Not sure it's a big deal though? Well, it's not a very big deal, but simplifies the code. Cheers, Alex -- -------------- next part -------------- A non-text attachment was scrubbed... Name: OpenPGP_signature Type: application/pgp-signature Size: 833 bytes Desc: OpenPGP digital signature URL: From xeioex at nginx.com Tue Jan 31 16:36:03 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 31 Jan 2023 16:36:03 +0000 Subject: [njs] 2023 year. Message-ID: details: https://hg.nginx.org/njs/rev/2a8800b7fa34 branches: changeset: 2032:2a8800b7fa34 user: Dmitry Volyntsev date: Mon Jan 30 17:35:58 2023 -0800 description: 2023 year. diffstat: LICENSE | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (15 lines): diff -r cba6c332ef5a -r 2a8800b7fa34 LICENSE --- a/LICENSE Fri Jan 27 15:19:38 2023 -0800 +++ b/LICENSE Mon Jan 30 17:35:58 2023 -0800 @@ -1,8 +1,9 @@ /* - * Copyright (C) 2015-2022 NGINX, Inc. + * Copyright (C) 2015-2023 NGINX, Inc. * Copyright (C) 2015-2021 Igor Sysoev - * Copyright (C) 2017-2022 Dmitry Volyntsev + * Copyright (C) 2017-2023 Dmitry Volyntsev * Copyright (C) 2019-2022 Alexander Borisov + * Copyright (C) 2022-2023 Vadim Zhestikov * All rights reserved. * * Redistribution and use in source and binary forms, with or without From xeioex at nginx.com Tue Jan 31 16:36:05 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 31 Jan 2023 16:36:05 +0000 Subject: [njs] Fixed typos introduced in 99b9f83e4d4d. Message-ID: details: https://hg.nginx.org/njs/rev/4f1e0dcd3c91 branches: changeset: 2033:4f1e0dcd3c91 user: Dmitry Volyntsev date: Mon Jan 30 17:43:59 2023 -0800 description: Fixed typos introduced in 99b9f83e4d4d. Found by Coverity (CID 1520597, 1520598, 1520599). diffstat: external/njs_xml_module.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (30 lines): diff -r 2a8800b7fa34 -r 4f1e0dcd3c91 external/njs_xml_module.c --- a/external/njs_xml_module.c Mon Jan 30 17:35:58 2023 -0800 +++ b/external/njs_xml_module.c Mon Jan 30 17:43:59 2023 -0800 @@ -1004,7 +1004,7 @@ njs_xml_ext_canonicalization(njs_vm_t *v nodes = xmlXPathNodeSetCreate(current); if (njs_slow_path(nodes == NULL)) { - njs_vm_memory_pool(vm); + njs_vm_memory_error(vm); goto error; } @@ -1184,7 +1184,7 @@ njs_xml_nset_create(njs_vm_t *vm, xmlDoc nset = njs_mp_zalloc(njs_vm_memory_pool(vm), sizeof(njs_xml_nset_t)); if (njs_slow_path(nset == NULL)) { - njs_vm_memory_pool(vm); + njs_vm_memory_error(vm); return NULL; } @@ -1213,7 +1213,7 @@ njs_xml_nset_children(njs_vm_t *vm, xmlN nodes = xmlXPathNodeSetCreate(parent); if (njs_slow_path(nodes == NULL)) { - njs_vm_memory_pool(vm); + njs_vm_memory_error(vm); return NULL; } From xeioex at nginx.com Tue Jan 31 16:36:07 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 31 Jan 2023 16:36:07 +0000 Subject: [njs] Fixed memory leak when exception happens at xml.exclusiveC14n(). Message-ID: details: https://hg.nginx.org/njs/rev/948b167202b2 branches: changeset: 2034:948b167202b2 user: Dmitry Volyntsev date: Mon Jan 30 18:19:16 2023 -0800 description: Fixed memory leak when exception happens at xml.exclusiveC14n(). Found by Coverity (CID 1520596). diffstat: external/njs_xml_module.c | 45 ++++++++++++++++----------------------------- src/test/njs_unit_test.c | 4 ++++ 2 files changed, 20 insertions(+), 29 deletions(-) diffs (136 lines): diff -r 4f1e0dcd3c91 -r 948b167202b2 external/njs_xml_module.c --- a/external/njs_xml_module.c Mon Jan 30 17:43:59 2023 -0800 +++ b/external/njs_xml_module.c Mon Jan 30 18:19:16 2023 -0800 @@ -69,8 +69,7 @@ static njs_int_t njs_xml_node_ext_text(n njs_value_t *value, njs_value_t *setval, njs_value_t *retval); static njs_xml_nset_t *njs_xml_nset_create(njs_vm_t *vm, xmlDoc *doc, - xmlNodeSet *nodes, njs_xml_nset_type_t type); -static njs_xml_nset_t *njs_xml_nset_children(njs_vm_t *vm, xmlNode *parent); + xmlNode *current, njs_xml_nset_type_t type); static njs_xml_nset_t *njs_xml_nset_add(njs_xml_nset_t *nset, njs_xml_nset_t *add); static void njs_xml_nset_cleanup(void *data); @@ -971,7 +970,6 @@ njs_xml_ext_canonicalization(njs_vm_t *v njs_str_t data, string; njs_chb_t chain; njs_bool_t comments; - xmlNodeSet *nodes; njs_value_t *excluding, *prefixes; njs_xml_doc_t *tree; njs_xml_nset_t *nset, *children; @@ -997,17 +995,11 @@ njs_xml_ext_canonicalization(njs_vm_t *v buf = xmlOutputBufferCreateIO(njs_xml_buf_write_cb, NULL, &chain, NULL); if (njs_slow_path(buf == NULL)) { njs_vm_error(vm, "xmlOutputBufferCreateIO() failed"); - goto error; + return NJS_ERROR; } comments = njs_value_bool(njs_arg(args, nargs, 3)); - nodes = xmlXPathNodeSetCreate(current); - if (njs_slow_path(nodes == NULL)) { - njs_vm_memory_error(vm); - goto error; - } - excluding = njs_arg(args, nargs, 2); if (!njs_value_is_null_or_undefined(excluding)) { @@ -1017,13 +1009,14 @@ njs_xml_ext_canonicalization(njs_vm_t *v goto error; } - nset = njs_xml_nset_create(vm, current->doc, nodes, + nset = njs_xml_nset_create(vm, current->doc, current, XML_NSET_TREE_NO_COMMENTS); if (njs_slow_path(nset == NULL)) { goto error; } - children = njs_xml_nset_children(vm, node); + children = njs_xml_nset_create(vm, node->doc, node, + XML_NSET_TREE_INVERT); if (njs_slow_path(children == NULL)) { goto error; } @@ -1031,7 +1024,7 @@ njs_xml_ext_canonicalization(njs_vm_t *v nset = njs_xml_nset_add(nset, children); } else { - nset = njs_xml_nset_create(vm, current->doc, nodes, + nset = njs_xml_nset_create(vm, current->doc, current, comments ? XML_NSET_TREE : XML_NSET_TREE_NO_COMMENTS); if (njs_slow_path(nset == NULL)) { @@ -1088,6 +1081,8 @@ njs_xml_ext_canonicalization(njs_vm_t *v error: + (void) xmlOutputBufferClose(buf); + njs_chb_destroy(&chain); return NJS_ERROR; @@ -1176,9 +1171,10 @@ njs_xml_attr_ext_prop_handler(njs_vm_t * static njs_xml_nset_t * -njs_xml_nset_create(njs_vm_t *vm, xmlDoc *doc, xmlNodeSet *nodes, +njs_xml_nset_create(njs_vm_t *vm, xmlDoc *doc, xmlNode *current, njs_xml_nset_type_t type) { + xmlNodeSet *nodes; njs_xml_nset_t *nset; njs_mp_cleanup_t *cln; @@ -1194,6 +1190,12 @@ njs_xml_nset_create(njs_vm_t *vm, xmlDoc return NULL; } + nodes = xmlXPathNodeSetCreate(current); + if (njs_slow_path(nodes == NULL)) { + njs_vm_memory_error(vm); + return NULL; + } + cln->handler = njs_xml_nset_cleanup; cln->data = nset; @@ -1207,21 +1209,6 @@ njs_xml_nset_create(njs_vm_t *vm, xmlDoc static njs_xml_nset_t * -njs_xml_nset_children(njs_vm_t *vm, xmlNode *parent) -{ - xmlNodeSet *nodes; - - nodes = xmlXPathNodeSetCreate(parent); - if (njs_slow_path(nodes == NULL)) { - njs_vm_memory_error(vm); - return NULL; - } - - return njs_xml_nset_create(vm, parent->doc, nodes, XML_NSET_TREE_INVERT); -} - - -static njs_xml_nset_t * njs_xml_nset_add(njs_xml_nset_t *nset, njs_xml_nset_t *add) { if (nset == NULL) { diff -r 4f1e0dcd3c91 -r 948b167202b2 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Mon Jan 30 17:43:59 2023 -0800 +++ b/src/test/njs_unit_test.c Mon Jan 30 18:19:16 2023 -0800 @@ -21545,6 +21545,10 @@ static njs_unit_test_t njs_xml_test[] = njs_str("{\"note\":{\"$name\":\"note\",\"$tags\":" "[{\"$name\":\"to\",\"$attrs\":{\"b\":\"bar\",\"a\":\"foo\"}," "\"$text\":\"Tove\"},{\"$name\":\"from\",\"$text\":\"Jani\"}]}}") }, + + { njs_str("var xml = require('xml');" + "var doc = xml.parse(``); xml.exclusiveC14n(doc, 1)"), + njs_str("Error: \"excluding\" argument is not a XMLNode object") }, };