From xeioex at nginx.com  Sat Jul  1 00:03:51 2023
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Sat, 01 Jul 2023 00:03:51 +0000
Subject: [njs] Tests: fixed benchmark after 57ca02d7404c.
Message-ID: <hg.494796d6d7f8.1688169831.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/494796d6d7f8
branches:  
changeset: 2171:494796d6d7f8
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Fri Jun 30 17:03:11 2023 -0700
description:
Tests: fixed benchmark after 57ca02d7404c.

diffstat:

 src/test/njs_benchmark.c |  15 +++++++++------
 1 files changed, 9 insertions(+), 6 deletions(-)

diffs (46 lines):

diff -r e85aaeaa7e55 -r 494796d6d7f8 src/test/njs_benchmark.c
--- a/src/test/njs_benchmark.c	Fri Jun 30 06:38:36 2023 -0700
+++ b/src/test/njs_benchmark.c	Fri Jun 30 17:03:11 2023 -0700
@@ -29,6 +29,12 @@ typedef struct {
 } njs_opts_t;
 
 
+njs_module_t *njs_benchmark_addon_external_modules[] = {
+    &njs_unit_test_external_module,
+    NULL,
+};
+
+
 static njs_int_t
 njs_benchmark_test(njs_vm_t *parent, njs_opts_t *opts, njs_value_t *report,
     njs_benchmark_test_t *test)
@@ -36,7 +42,7 @@ njs_benchmark_test(njs_vm_t *parent, njs
     u_char        *start;
     njs_vm_t      *vm, *nvm;
     uint64_t      ns;
-    njs_int_t     ret, proto_id;
+    njs_int_t     ret;
     njs_str_t     s, *expected;
     njs_uint_t    i, n;
     njs_bool_t    success;
@@ -49,6 +55,8 @@ njs_benchmark_test(njs_vm_t *parent, njs
 
     njs_vm_opt_init(&options);
 
+    options.addons = njs_benchmark_addon_external_modules;
+
     vm = NULL;
     nvm = NULL;
     ret = NJS_ERROR;
@@ -67,11 +75,6 @@ njs_benchmark_test(njs_vm_t *parent, njs
         goto done;
     }
 
-    proto_id = njs_externals_shared_init(vm);
-    if (proto_id < 0) {
-        goto done;
-    }
-
     n = test->repeat;
     expected = &test->result;
 

From mdounin at mdounin.ru  Sat Jul  1 00:21:55 2023
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Sat, 01 Jul 2023 03:21:55 +0300
Subject: [PATCH 0 of 2] h2_http2.t fixes
Message-ID: <patchbomb.1688170915@vm-bsd.mdounin.ru>

Hello!

Here is a couple of fixes for h2_http2.t to make it work with
various old OpenSSL versions.

-- 
Maxim Dounin

From mdounin at mdounin.ru  Sat Jul  1 00:21:56 2023
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Sat, 01 Jul 2023 03:21:56 +0300
Subject: [PATCH 1 of 2] Tests: fixed h2_http2.t when nginx compiled without
 ALPN support
In-Reply-To: <patchbomb.1688170915@vm-bsd.mdounin.ru>
References: <patchbomb.1688170915@vm-bsd.mdounin.ru>
Message-ID: <bf9961a4507784b66965.1688170916@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1688164311 -10800
#      Sat Jul 01 01:31:51 2023 +0300
# Node ID bf9961a4507784b669650e7ef1d3cbacce8c94e1
# Parent  22f45bf99a9e39e02d8ce07532d2cbfc89e7d8d1
Tests: fixed h2_http2.t when nginx compiled without ALPN support.

ALPN support is only available with OpenSSL 1.0.2 or newer.

diff --git a/h2_http2.t b/h2_http2.t
--- a/h2_http2.t
+++ b/h2_http2.t
@@ -127,9 +127,16 @@ ok(!get_ssl_socket(8443, 'disabled'), 's
 
 }
 
+TODO: {
+local $TODO = 'OpenSSL too old'
+	if $t->has_module('OpenSSL')
+	and not $t->has_feature('openssl:1.0.2');
+
 is(get_https(8443, 'http2'), 200, 'host to enabled');
 is(get_https(8443, 'disabled', 'http2'), 421, 'host to disabled');
 
+}
+
 # make sure HTTP/2 can be enabled selectively on virtual servers
 
 TODO: {

From mdounin at mdounin.ru  Sat Jul  1 00:21:57 2023
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Sat, 01 Jul 2023 03:21:57 +0300
Subject: [PATCH 2 of 2] Tests: adjusted TODO for OpenSSL 1.0.2h and up in
 h2_http2.t
In-Reply-To: <patchbomb.1688170915@vm-bsd.mdounin.ru>
References: <patchbomb.1688170915@vm-bsd.mdounin.ru>
Message-ID: <5ef10e454094ad5d2315.1688170917@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1688164312 -10800
#      Sat Jul 01 01:31:52 2023 +0300
# Node ID 5ef10e454094ad5d231552f0fc2c386e9cc33585
# Parent  bf9961a4507784b669650e7ef1d3cbacce8c94e1
Tests: adjusted TODO for OpenSSL 1.0.2h and up in h2_http2.t.

OpenSSL uses correct SNI/ALPN callback order (SNI callback before ALPN
callback) starting with OpenSSL 1.0.2h, so "sni to enabled" test
is expected to succeed starting with OpenSSL 1.0.2h.

With this change, the "openssl:..." feature test now supports checking
patch level encoded as letters, such as in "openssl:1.0.2h".

diff --git a/h2_http2.t b/h2_http2.t
--- a/h2_http2.t
+++ b/h2_http2.t
@@ -151,9 +151,9 @@ ok(!get_ssl_socket(8444), 'default to di
 TODO: {
 local $TODO = 'broken ALPN/SNI order in LibreSSL'
 	if $t->has_module('LibreSSL');
-local $TODO = 'OpenSSL too old'
+local $TODO = 'broken ALPN/SNI order in OpenSSL before 1.0.2h'
 	if $t->has_module('OpenSSL')
-	and not $t->has_feature('openssl:1.1.0');
+	and not $t->has_feature('openssl:1.0.2h');
 
 is(get_https(8444, 'http2'), 200, 'sni to enabled');
 
diff --git a/lib/Test/Nginx.pm b/lib/Test/Nginx.pm
--- a/lib/Test/Nginx.pm
+++ b/lib/Test/Nginx.pm
@@ -266,20 +266,22 @@ sub has_feature($) {
 		return 0;
 	}
 
-	if ($feature =~ /^(openssl|libressl):([0-9.]+)/) {
+	if ($feature =~ /^(openssl|libressl):([0-9.]+)([a-z]*)/) {
 		my $library = $1;
 		my $need = $2;
+		my $patch = $3;
 
 		$self->{_configure_args} = `$NGINX -V 2>&1`
 			if !defined $self->{_configure_args};
 
 		return 0 unless
-			$self->{_configure_args} =~ /with $library ([0-9.]+)/i;
+			$self->{_configure_args}
+			=~ /with $library ([0-9.]+)([a-z]*)/i;
 
-		my @v = split(/\./, $1);
+		my @v = (split(/\./, $1), unpack("C*", $2));
 		my ($n, $v);
 
-		for $n (split(/\./, $need)) {
+		for $n (split(/\./, $need), unpack("C*", $patch)) {
 			$v = shift @v || 0;
 			return 0 if $n > $v;
 			return 1 if $v > $n;

From v.zhestikov at f5.com  Sat Jul  1 02:51:11 2023
From: v.zhestikov at f5.com (Vadim Zhestikov)
Date: Sat, 01 Jul 2023 02:51:11 +0000
Subject: [njs] Fixed parsing of invalid for statement.
Message-ID: <hg.2c532e7c29ac.1688179871.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/2c532e7c29ac
branches:  
changeset: 2172:2c532e7c29ac
user:      Vadim Zhestikov <v.zhestikov at f5.com>
date:      Fri Jun 30 19:49:45 2023 -0700
description:
Fixed parsing of invalid for statement.

diffstat:

 src/njs_parser.c         |  16 ++++++++++++++--
 src/test/njs_unit_test.c |   3 +++
 2 files changed, 17 insertions(+), 2 deletions(-)

diffs (46 lines):

diff -r 494796d6d7f8 -r 2c532e7c29ac src/njs_parser.c
--- a/src/njs_parser.c	Fri Jun 30 17:03:11 2023 -0700
+++ b/src/njs_parser.c	Fri Jun 30 19:49:45 2023 -0700
@@ -5491,6 +5491,18 @@ njs_parser_iteration_statement_for(njs_p
 
 
 static njs_int_t
+njs_parser_for_var_in_of_expression_chk_fail(njs_parser_t *parser,
+    njs_lexer_token_t *token, njs_queue_link_t *current)
+{
+    if (parser->ret != NJS_OK) {
+        return njs_parser_failed(parser);
+    }
+
+    return njs_parser_for_var_in_of_expression(parser, token, current);
+}
+
+
+static njs_int_t
 njs_parser_for_expression_map_reparse(njs_parser_t *parser,
     njs_lexer_token_t *token, njs_queue_link_t *current)
 {
@@ -5517,8 +5529,8 @@ njs_parser_for_expression_map_reparse(nj
 
         *text = token->text;
 
-        return njs_parser_after(parser, current, text, 1,
-                                njs_parser_for_var_in_of_expression);
+        return njs_parser_after(parser, current, text, 0,
+                                njs_parser_for_var_in_of_expression_chk_fail);
     }
 
     return njs_parser_stack_pop(parser);
diff -r 494796d6d7f8 -r 2c532e7c29ac src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Fri Jun 30 17:03:11 2023 -0700
+++ b/src/test/njs_unit_test.c	Fri Jun 30 19:49:45 2023 -0700
@@ -2975,6 +2975,9 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("for(var``>0; 0 ;) ;"),
       njs_str("SyntaxError: Unexpected token \"`\" in 1") },
 
+    { njs_str("for(i;;)for(-new+3;;)break;"),
+      njs_str("SyntaxError: Unexpected token \"+\" in 1") },
+
     /* switch. */
 
     { njs_str("switch"),

From v.zhestikov at f5.com  Sat Jul  1 02:51:13 2023
From: v.zhestikov at f5.com (Vadim Zhestikov)
Date: Sat, 01 Jul 2023 02:51:13 +0000
Subject: [njs] Improved interactive shell.
Message-ID: <hg.0e97da4147da.1688179873.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/0e97da4147da
branches:  
changeset: 2173:0e97da4147da
user:      Vadim Zhestikov <v.zhestikov at f5.com>
date:      Fri Jun 30 19:49:46 2023 -0700
description:
Improved interactive shell.

diffstat:

 external/njs_shell.c |  17 ++++++++++++-----
 1 files changed, 12 insertions(+), 5 deletions(-)

diffs (42 lines):

diff -r 2c532e7c29ac -r 0e97da4147da external/njs_shell.c
--- a/external/njs_shell.c	Fri Jun 30 19:49:45 2023 -0700
+++ b/external/njs_shell.c	Fri Jun 30 19:49:46 2023 -0700
@@ -1073,19 +1073,24 @@ njs_cb_line_handler(char *line_in)
     njs_int_t  ret;
     njs_str_t  line;
 
-    if (line_in == NULL || strcmp(line_in, ".exit") == 0) {
+    if (line_in == NULL) {
         njs_running = NJS_DONE;
         return;
     }
 
+    line.start = (u_char *) line_in;
+    line.length = njs_strlen(line.start);
+
+    if (strcmp(line_in, ".exit") == 0) {
+        njs_running = NJS_DONE;
+        goto free_line;
+    }
+
     njs_sigint_count = 0;
 
-    line.start = (u_char *) line_in;
-    line.length = njs_strlen(line.start);
-
     if (line.length == 0) {
         rl_callback_handler_install(">> ", njs_cb_line_handler);
-        return;
+        goto free_line;
     }
 
     add_history((char *) line.start);
@@ -1099,6 +1104,8 @@ njs_cb_line_handler(char *line_in)
         rl_callback_handler_install(">> ", njs_cb_line_handler);
     }
 
+free_line:
+
     free(line.start);
 }
 

From xeioex at nginx.com  Sat Jul  1 04:03:32 2023
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Sat, 01 Jul 2023 04:03:32 +0000
Subject: [njs] Tests: improved memory sanitizer build.
Message-ID: <hg.3386684745b7.1688184212.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/3386684745b7
branches:  
changeset: 2174:3386684745b7
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Fri Jun 30 21:02:44 2023 -0700
description:
Tests: improved memory sanitizer build.

Disable webcrypto tests because they required instrumented openssl.

diffstat:

 src/test/njs_unit_test.c |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (12 lines):

diff -r 0e97da4147da -r 3386684745b7 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Fri Jun 30 19:49:46 2023 -0700
+++ b/src/test/njs_unit_test.c	Fri Jun 30 21:02:44 2023 -0700
@@ -25090,7 +25090,7 @@ static njs_test_suite_t  njs_suites[] =
       njs_disabled_denormals_tests },
 
     {
-#if (NJS_HAVE_OPENSSL)
+#if (NJS_HAVE_OPENSSL && !NJS_HAVE_MEMORY_SANITIZER)
         njs_str("webcrypto"),
 #else
         njs_str(""),

From xeioex at nginx.com  Mon Jul  3 20:34:11 2023
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Mon, 03 Jul 2023 20:34:11 +0000
Subject: [njs] Added njs_vm_external_constructor().
Message-ID: <hg.167f75576f49.1688416451.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/167f75576f49
branches:  
changeset: 2175:167f75576f49
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Mon Jul 03 12:49:00 2023 -0700
description:
Added njs_vm_external_constructor().

The new API allows to add new constructor/prototype pairs.

diffstat:

 external/njs_crypto_module.c       |    1 +
 external/njs_fs_module.c           |    1 +
 external/njs_query_string_module.c |    1 +
 external/njs_shell.c               |    1 +
 external/njs_webcrypto_module.c    |    1 +
 external/njs_xml_module.c          |    1 +
 external/njs_zlib_module.c         |    1 +
 nginx/ngx_http_js_module.c         |    1 +
 nginx/ngx_js.c                     |    1 +
 nginx/ngx_js_fetch.c               |    1 +
 nginx/ngx_stream_js_module.c       |    1 +
 src/njs.h                          |   18 ++-
 src/njs_array.c                    |    2 +-
 src/njs_array_buffer.c             |    2 +-
 src/njs_async.c                    |    2 +-
 src/njs_buffer.c                   |   13 +-
 src/njs_builtin.c                  |  127 ++++----------------
 src/njs_date.c                     |    2 +-
 src/njs_error.c                    |   16 +-
 src/njs_extern.c                   |   71 +++++++++++
 src/njs_function.c                 |   10 +-
 src/njs_json.c                     |    2 +-
 src/njs_object.c                   |   16 +-
 src/njs_parser.c                   |    6 +-
 src/njs_promise.c                  |   12 +-
 src/njs_regexp.c                   |    8 +-
 src/njs_typed_array.c              |    6 +-
 src/njs_value.c                    |    2 +-
 src/njs_value.h                    |    5 +-
 src/njs_vm.c                       |  224 ++++++++++++++++++++++++++++++++++--
 src/njs_vm.h                       |   34 ++--
 src/njs_vmcode.c                   |    2 +-
 src/test/njs_externals_test.c      |  113 ++++++++++++++++++-
 src/test/njs_unit_test.c           |   44 +++++++-
 34 files changed, 561 insertions(+), 187 deletions(-)

diffs (truncated from 1548 to 1000 lines):

diff -r 3386684745b7 -r 167f75576f49 external/njs_crypto_module.c
--- a/external/njs_crypto_module.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/external/njs_crypto_module.c	Mon Jul 03 12:49:00 2023 -0700
@@ -287,6 +287,7 @@ static njs_int_t    njs_crypto_hmac_prot
 
 njs_module_t  njs_crypto_module = {
     .name = njs_str("crypto"),
+    .preinit = NULL,
     .init = njs_crypto_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 external/njs_fs_module.c
--- a/external/njs_fs_module.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/external/njs_fs_module.c	Mon Jul 03 12:49:00 2023 -0700
@@ -1407,6 +1407,7 @@ static njs_int_t    njs_fs_bytes_written
 
 njs_module_t  njs_fs_module = {
     .name = njs_str("fs"),
+    .preinit = NULL,
     .init = njs_fs_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 external/njs_query_string_module.c
--- a/external/njs_query_string_module.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/external/njs_query_string_module.c	Mon Jul 03 12:49:00 2023 -0700
@@ -105,6 +105,7 @@ static njs_external_t  njs_ext_query_str
 
 njs_module_t  njs_query_string_module = {
     .name = njs_str("querystring"),
+    .preinit = NULL,
     .init = njs_query_string_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 external/njs_shell.c
--- a/external/njs_shell.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/external/njs_shell.c	Mon Jul 03 12:49:00 2023 -0700
@@ -254,6 +254,7 @@ static njs_vm_ops_t njs_console_ops = {
 
 njs_module_t  njs_console_module = {
     .name = njs_str("console"),
+    .preinit = NULL,
     .init = njs_externals_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 external/njs_webcrypto_module.c
--- a/external/njs_webcrypto_module.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/external/njs_webcrypto_module.c	Mon Jul 03 12:49:00 2023 -0700
@@ -659,6 +659,7 @@ static njs_external_t  njs_ext_webcrypto
 
 njs_module_t  njs_webcrypto_module = {
     .name = njs_str("webcrypto"),
+    .preinit = NULL,
     .init = njs_webcrypto_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 external/njs_xml_module.c
--- a/external/njs_xml_module.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/external/njs_xml_module.c	Mon Jul 03 12:49:00 2023 -0700
@@ -402,6 +402,7 @@ static njs_external_t  njs_ext_xml_attr[
 
 njs_module_t  njs_xml_module = {
     .name = njs_str("xml"),
+    .preinit = NULL,
     .init = njs_xml_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 external/njs_zlib_module.c
--- a/external/njs_zlib_module.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/external/njs_zlib_module.c	Mon Jul 03 12:49:00 2023 -0700
@@ -176,6 +176,7 @@ static njs_external_t  njs_ext_zlib[] = 
 
 njs_module_t  njs_zlib_module = {
     .name = njs_str("zlib"),
+    .preinit = NULL,
     .init = njs_zlib_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 nginx/ngx_http_js_module.c
--- a/nginx/ngx_http_js_module.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/nginx/ngx_http_js_module.c	Mon Jul 03 12:49:00 2023 -0700
@@ -786,6 +786,7 @@ static njs_vm_meta_t ngx_http_js_metas =
 
 njs_module_t  ngx_js_http_module = {
     .name = njs_str("http"),
+    .preinit = NULL,
     .init = ngx_js_http_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 nginx/ngx_js.c
--- a/nginx/ngx_js.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/nginx/ngx_js.c	Mon Jul 03 12:49:00 2023 -0700
@@ -165,6 +165,7 @@ static njs_external_t  ngx_js_ext_core[]
 
 njs_module_t  ngx_js_ngx_module = {
     .name = njs_str("ngx"),
+    .preinit = NULL,
     .init = ngx_js_core_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 nginx/ngx_js_fetch.c
--- a/nginx/ngx_js_fetch.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/nginx/ngx_js_fetch.c	Mon Jul 03 12:49:00 2023 -0700
@@ -656,6 +656,7 @@ static njs_int_t    ngx_http_js_fetch_he
 
 njs_module_t  ngx_js_fetch_module = {
     .name = njs_str("fetch"),
+    .preinit = NULL,
     .init = ngx_js_fetch_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 nginx/ngx_stream_js_module.c
--- a/nginx/ngx_stream_js_module.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/nginx/ngx_stream_js_module.c	Mon Jul 03 12:49:00 2023 -0700
@@ -568,6 +568,7 @@ static njs_int_t    ngx_stream_js_sessio
 
 njs_module_t  ngx_js_stream_module = {
     .name = njs_str("stream"),
+    .preinit = NULL,
     .init = ngx_js_stream_init,
 };
 
diff -r 3386684745b7 -r 167f75576f49 src/njs.h
--- a/src/njs.h	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs.h	Mon Jul 03 12:49:00 2023 -0700
@@ -257,6 +257,7 @@ typedef njs_int_t (*njs_addon_init_pt)(n
 
 typedef struct {
     njs_str_t                       name;
+    njs_addon_init_pt               preinit;
     njs_addon_init_pt               init;
 } njs_module_t;
 
@@ -403,6 +404,10 @@ NJS_EXPORT njs_int_t njs_vm_add_path(njs
 
 NJS_EXPORT njs_int_t njs_vm_external_prototype(njs_vm_t *vm,
     const njs_external_t *definition, njs_uint_t n);
+NJS_EXPORT njs_int_t njs_vm_external_constructor(njs_vm_t *vm,
+    const njs_str_t *name, njs_function_native_t native,
+    const njs_external_t *ctor_props, njs_uint_t ctor_nprops,
+    const njs_external_t *proto_props, njs_uint_t proto_nprops);
 NJS_EXPORT njs_int_t njs_vm_external_create(njs_vm_t *vm, njs_value_t *value,
     njs_int_t proto_id, njs_external_ptr_t external, njs_bool_t shared);
 NJS_EXPORT njs_external_ptr_t njs_vm_external(njs_vm_t *vm,
@@ -417,6 +422,15 @@ NJS_EXPORT njs_int_t njs_value_property_
 NJS_EXPORT uintptr_t njs_vm_meta(njs_vm_t *vm, njs_uint_t index);
 NJS_EXPORT njs_vm_opt_t *njs_vm_options(njs_vm_t *vm);
 
+NJS_EXPORT njs_int_t njs_error_constructor(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t type, njs_value_t *retval);
+NJS_EXPORT njs_int_t njs_object_prototype_create_constructor(njs_vm_t *vm,
+    njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval,
+    njs_value_t *retval);
+NJS_EXPORT njs_int_t njs_object_prototype_create(njs_vm_t *vm,
+    njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval,
+    njs_value_t *retval);
+
 NJS_EXPORT njs_function_t *njs_vm_function_alloc(njs_vm_t *vm,
     njs_function_native_t native, njs_bool_t shared, njs_bool_t ctor);
 
@@ -433,7 +447,9 @@ NJS_EXPORT njs_function_t *njs_vm_functi
 NJS_EXPORT njs_bool_t njs_vm_constructor(njs_vm_t *vm);
 
 NJS_EXPORT void njs_vm_throw(njs_vm_t *vm, const njs_value_t *value);
-NJS_EXPORT void njs_vm_error2(njs_vm_t *vm, unsigned type, const char *fmt,
+NJS_EXPORT void njs_vm_error2(njs_vm_t *vm, unsigned error_type,
+    const char *fmt, ...);
+NJS_EXPORT void njs_vm_error3(njs_vm_t *vm, unsigned type, const char *fmt,
     ...);
 NJS_EXPORT void njs_vm_exception_get(njs_vm_t *vm, njs_value_t *retval);
 NJS_EXPORT njs_mp_t *njs_vm_memory_pool(njs_vm_t *vm);
diff -r 3386684745b7 -r 167f75576f49 src/njs_array.c
--- a/src/njs_array.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_array.c	Mon Jul 03 12:49:00 2023 -0700
@@ -83,7 +83,7 @@ njs_array_alloc(njs_vm_t *vm, njs_bool_t
     array->start = array->data;
     njs_lvlhsh_init(&array->object.hash);
     array->object.shared_hash = vm->shared->array_instance_hash;
-    array->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_ARRAY].object;
+    array->object.__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_ARRAY);
     array->object.slots = NULL;
     array->object.type = NJS_ARRAY;
     array->object.shared = 0;
diff -r 3386684745b7 -r 167f75576f49 src/njs_array_buffer.c
--- a/src/njs_array_buffer.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_array_buffer.c	Mon Jul 03 12:49:00 2023 -0700
@@ -33,7 +33,7 @@ njs_array_buffer_alloc(njs_vm_t *vm, uin
         goto memory_error;
     }
 
-    proto = &vm->prototypes[NJS_OBJ_TYPE_ARRAY_BUFFER].object;
+    proto = njs_vm_proto(vm, NJS_OBJ_TYPE_ARRAY_BUFFER);
 
     njs_lvlhsh_init(&array->object.hash);
     njs_lvlhsh_init(&array->object.shared_hash);
diff -r 3386684745b7 -r 167f75576f49 src/njs_async.c
--- a/src/njs_async.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_async.c	Mon Jul 03 12:49:00 2023 -0700
@@ -18,7 +18,7 @@ njs_async_function_frame_invoke(njs_vm_t
     njs_value_t               ctor;
     njs_promise_capability_t  *capability;
 
-    njs_set_function(&ctor, &vm->constructors[NJS_OBJ_TYPE_PROMISE]);
+    njs_set_function(&ctor, &njs_vm_ctor(vm, NJS_OBJ_TYPE_PROMISE));
 
     capability = njs_promise_new_capability(vm, &ctor);
     if (njs_slow_path(capability == NULL)) {
diff -r 3386684745b7 -r 167f75576f49 src/njs_buffer.c
--- a/src/njs_buffer.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_buffer.c	Mon Jul 03 12:49:00 2023 -0700
@@ -126,6 +126,7 @@ static njs_external_t  njs_ext_buffer[] 
 
 njs_module_t  njs_buffer_module = {
     .name = njs_str("buffer"),
+    .preinit = NULL,
     .init = njs_buffer_init,
 };
 
@@ -147,7 +148,7 @@ njs_buffer_set(njs_vm_t *vm, njs_value_t
 
     buffer = (njs_array_buffer_t *) &array[1];
 
-    proto = &vm->prototypes[NJS_OBJ_TYPE_ARRAY_BUFFER].object;
+    proto = njs_vm_proto(vm, NJS_OBJ_TYPE_ARRAY_BUFFER);
 
     njs_lvlhsh_init(&buffer->object.hash);
     njs_lvlhsh_init(&buffer->object.shared_hash);
@@ -161,7 +162,7 @@ njs_buffer_set(njs_vm_t *vm, njs_value_t
     buffer->u.data = (void *) start;
     buffer->size = size;
 
-    proto = &vm->prototypes[NJS_OBJ_TYPE_BUFFER].object;
+    proto = njs_vm_proto(vm, NJS_OBJ_TYPE_BUFFER);
 
     array->type = NJS_OBJ_TYPE_UINT8_ARRAY;
     njs_lvlhsh_init(&array->object.hash);
@@ -197,7 +198,7 @@ njs_buffer_alloc(njs_vm_t *vm, size_t si
         return NULL;
     }
 
-    array->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_BUFFER].object;
+    array->object.__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_BUFFER);
 
     return array;
 }
@@ -462,7 +463,7 @@ njs_buffer_from_array_buffer(njs_vm_t *v
         return NJS_ERROR;
     }
 
-    buffer->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_BUFFER].object;
+    buffer->object.__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_BUFFER);
 
     buffer->offset = off;
     buffer->byte_length = len;
@@ -943,7 +944,7 @@ njs_buffer_is_buffer(njs_vm_t *vm, njs_v
     array = njs_buffer_slot_internal(vm, njs_arg(args, nargs, 1));
 
     if (njs_fast_path(array != NULL && array->object.__proto__
-                      == &vm->prototypes[NJS_OBJ_TYPE_BUFFER].object))
+                      == njs_vm_proto(vm, NJS_OBJ_TYPE_BUFFER)))
     {
         is = 1;
     }
@@ -2184,7 +2185,7 @@ njs_buffer_prototype_slice(njs_vm_t *vm,
     }
 
     array = njs_typed_array(retval);
-    array->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_BUFFER].object;
+    array->object.__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_BUFFER);
 
     return NJS_OK;
 }
diff -r 3386684745b7 -r 167f75576f49 src/njs_builtin.c
--- a/src/njs_builtin.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_builtin.c	Mon Jul 03 12:49:00 2023 -0700
@@ -115,7 +115,7 @@ njs_object_hash_init(njs_vm_t *vm, njs_l
 njs_int_t
 njs_builtin_objects_create(njs_vm_t *vm)
 {
-    njs_int_t                  ret;
+    njs_int_t                  ret, index;
     njs_uint_t                 i;
     njs_object_t               *object, *string_object;
     njs_function_t             *constructor;
@@ -129,6 +129,8 @@ njs_builtin_objects_create(njs_vm_t *vm)
         return NJS_ERROR;
     }
 
+    vm->shared = shared;
+
     njs_lvlhsh_init(&shared->keywords_hash);
     njs_lvlhsh_init(&shared->values_hash);
 
@@ -204,34 +206,42 @@ njs_builtin_objects_create(njs_vm_t *vm)
         return NJS_ERROR;
     }
 
-    prototype = shared->prototypes;
+    for (i = NJS_OBJ_TYPE_OBJECT; i < NJS_OBJ_TYPE_MAX; i++) {
+        index = njs_vm_ctor_push(vm);
+        if (njs_slow_path(index < 0)) {
+            return NJS_ERROR;
+        }
 
-    for (i = NJS_OBJ_TYPE_OBJECT; i < NJS_OBJ_TYPE_MAX; i++) {
-        prototype[i] = njs_object_type_init[i]->prototype_value;
+        njs_assert_msg((njs_uint_t) index == i,
+                       "ctor index should match object type");
 
-        ret = njs_object_hash_init(vm, &prototype[i].object.shared_hash,
+        prototype = njs_shared_prototype(shared, i);
+        *prototype = njs_object_type_init[i]->prototype_value;
+
+        ret = njs_object_hash_init(vm, &prototype->object.shared_hash,
                                    njs_object_type_init[i]->prototype_props);
         if (njs_slow_path(ret != NJS_OK)) {
             return NJS_ERROR;
         }
 
-        prototype[i].object.extensible = 1;
+        prototype->object.extensible = 1;
     }
 
-    shared->prototypes[NJS_OBJ_TYPE_REGEXP].regexp.pattern =
-                                              shared->empty_regexp_pattern;
-
-    constructor = shared->constructors;
+    prototype = njs_shared_prototype(shared, NJS_OBJ_TYPE_REGEXP);
+    prototype->regexp.pattern = shared->empty_regexp_pattern;
 
     for (i = NJS_OBJ_TYPE_OBJECT; i < NJS_OBJ_TYPE_MAX; i++) {
+        constructor = njs_shared_ctor(shared, i);
+
         if (njs_object_type_init[i]->constructor_props == NULL) {
+            njs_memzero(constructor, sizeof(njs_function_t));
             continue;
         }
 
-        constructor[i] = njs_object_type_init[i]->constructor;
-        constructor[i].object.shared = 0;
+        *constructor = njs_object_type_init[i]->constructor;
+        constructor->object.shared = 0;
 
-        ret = njs_object_hash_init(vm, &constructor[i].object.shared_hash,
+        ret = njs_object_hash_init(vm, &constructor->object.shared_hash,
                                    njs_object_type_init[i]->constructor_props);
         if (njs_slow_path(ret != NJS_OK)) {
             return NJS_ERROR;
@@ -259,91 +269,6 @@ njs_builtin_objects_create(njs_vm_t *vm)
 
     njs_lvlhsh_init(&shared->modules_hash);
 
-    vm->shared = shared;
-
-    return NJS_OK;
-}
-
-
-njs_int_t
-njs_builtin_objects_clone(njs_vm_t *vm, njs_value_t *global)
-{
-    size_t        size;
-    njs_uint_t    i;
-    njs_object_t  *object_prototype, *function_prototype,
-                  *typed_array_prototype, *error_prototype, *async_prototype,
-                  *typed_array_ctor, *error_ctor;
-
-    /*
-     * Copy both prototypes and constructors arrays by one memcpy()
-     * because they are stored together.
-     */
-    size = (sizeof(njs_object_prototype_t) + sizeof(njs_function_t))
-           * NJS_OBJ_TYPE_MAX;
-
-    memcpy(vm->prototypes, vm->shared->prototypes, size);
-
-    object_prototype = &vm->prototypes[NJS_OBJ_TYPE_OBJECT].object;
-
-    for (i = NJS_OBJ_TYPE_ARRAY; i < NJS_OBJ_TYPE_NORMAL_MAX; i++) {
-        vm->prototypes[i].object.__proto__ = object_prototype;
-    }
-
-    typed_array_prototype = &vm->prototypes[NJS_OBJ_TYPE_TYPED_ARRAY].object;
-
-    for (i = NJS_OBJ_TYPE_TYPED_ARRAY_MIN;
-         i < NJS_OBJ_TYPE_TYPED_ARRAY_MAX;
-         i++)
-    {
-        vm->prototypes[i].object.__proto__ = typed_array_prototype;
-    }
-
-    vm->prototypes[NJS_OBJ_TYPE_ARRAY_ITERATOR].object.__proto__ =
-                              &vm->prototypes[NJS_OBJ_TYPE_ITERATOR].object;
-
-    vm->prototypes[NJS_OBJ_TYPE_BUFFER].object.__proto__ =
-                              &vm->prototypes[NJS_OBJ_TYPE_UINT8_ARRAY].object;
-
-    error_prototype = &vm->prototypes[NJS_OBJ_TYPE_ERROR].object;
-    error_prototype->__proto__ = object_prototype;
-
-    for (i = NJS_OBJ_TYPE_EVAL_ERROR; i < NJS_OBJ_TYPE_MAX; i++) {
-        vm->prototypes[i].object.__proto__ = error_prototype;
-    }
-
-    function_prototype = &vm->prototypes[NJS_OBJ_TYPE_FUNCTION].object;
-
-    async_prototype = &vm->prototypes[NJS_OBJ_TYPE_ASYNC_FUNCTION].object;
-    async_prototype->__proto__ = function_prototype;
-
-    for (i = NJS_OBJ_TYPE_OBJECT; i < NJS_OBJ_TYPE_NORMAL_MAX; i++) {
-        vm->constructors[i].object.__proto__ = function_prototype;
-    }
-
-    typed_array_ctor = &vm->constructors[NJS_OBJ_TYPE_TYPED_ARRAY].object;
-
-    for (i = NJS_OBJ_TYPE_TYPED_ARRAY_MIN;
-         i < NJS_OBJ_TYPE_TYPED_ARRAY_MAX;
-         i++)
-    {
-        vm->constructors[i].object.__proto__ = typed_array_ctor;
-    }
-
-    error_ctor = &vm->constructors[NJS_OBJ_TYPE_ERROR].object;
-    error_ctor->__proto__ = function_prototype;
-
-    for (i = NJS_OBJ_TYPE_EVAL_ERROR; i < NJS_OBJ_TYPE_MAX; i++) {
-        vm->constructors[i].object.__proto__ = error_ctor;
-    }
-
-    vm->global_object.__proto__ = object_prototype;
-
-    njs_set_undefined(global);
-    njs_set_object(global, &vm->global_object);
-
-    vm->string_object = vm->shared->string_object;
-    vm->string_object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_STRING].object;
-
     return NJS_OK;
 }
 
@@ -838,7 +763,7 @@ njs_builtin_match_native_function(njs_vm
     /* Constructor from built-in modules (not-mapped to global object). */
 
     for (i = NJS_OBJ_TYPE_HIDDEN_MIN; i < NJS_OBJ_TYPE_HIDDEN_MAX; i++) {
-        njs_set_object(&value, &vm->constructors[i].object);
+        njs_set_object(&value, &njs_vm_ctor(vm, i).object);
 
         ret = njs_value_property(vm, &value, njs_value_arg(&njs_string_name),
                                  &tag);
@@ -847,7 +772,7 @@ njs_builtin_match_native_function(njs_vm
             njs_string_get(&tag, &ctx.match);
         }
 
-        ret = njs_object_traverse(vm, &vm->constructors[i].object, &ctx,
+        ret = njs_object_traverse(vm, njs_object(&value), &ctx,
                                   njs_builtin_traverse);
 
         if (ret == NJS_DONE) {
@@ -1253,7 +1178,7 @@ njs_top_level_constructor(njs_vm_t *vm, 
             return NJS_DECLINED;
         }
 
-        ctor = &vm->constructors[njs_prop_magic16(self)];
+        ctor = &njs_vm_ctor(vm, njs_prop_magic16(self));
 
         njs_set_function(retval, ctor);
     }
diff -r 3386684745b7 -r 167f75576f49 src/njs_date.c
--- a/src/njs_date.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_date.c	Mon Jul 03 12:49:00 2023 -0700
@@ -364,7 +364,7 @@ njs_date_alloc(njs_vm_t *vm, double time
     date->object.extensible = 1;
     date->object.error_data = 0;
     date->object.fast_array = 0;
-    date->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_DATE].object;
+    date->object.__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_DATE);
     date->object.slots = NULL;
 
     date->time = time;
diff -r 3386684745b7 -r 167f75576f49 src/njs_error.c
--- a/src/njs_error.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_error.c	Mon Jul 03 12:49:00 2023 -0700
@@ -76,7 +76,7 @@ njs_throw_error(njs_vm_t *vm, njs_object
     va_list  args;
 
     va_start(args, fmt);
-    njs_throw_error_va(vm, &vm->prototypes[type].object, fmt, args);
+    njs_throw_error_va(vm, njs_vm_proto(vm, type), fmt, args);
     va_end(args);
 }
 
@@ -96,7 +96,7 @@ njs_error_fmt_new(njs_vm_t *vm, njs_valu
         va_end(args);
     }
 
-    njs_error_new(vm, dst, &vm->prototypes[type].object, buf, p - buf);
+    njs_error_new(vm, dst, njs_vm_proto(vm, type), buf, p - buf);
 }
 
 
@@ -298,7 +298,7 @@ memory_error:
 }
 
 
-static njs_int_t
+njs_int_t
 njs_error_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     njs_index_t type, njs_value_t *retval)
 {
@@ -339,7 +339,7 @@ njs_error_constructor(njs_vm_t *vm, njs_
         }
     }
 
-    error = njs_error_alloc(vm, &vm->prototypes[type].object, NULL,
+    error = njs_error_alloc(vm, njs_vm_proto(vm, type), NULL,
                             njs_is_defined(value) ? value : NULL,
                             njs_is_defined(&list) ? &list : NULL);
     if (njs_slow_path(error == NULL)) {
@@ -499,15 +499,13 @@ const njs_object_init_t  njs_aggregate_e
 void
 njs_memory_error_set(njs_vm_t *vm, njs_value_t *value)
 {
-    njs_object_t            *object;
-    njs_object_prototype_t  *prototypes;
+    njs_object_t  *object;
 
-    prototypes = vm->prototypes;
     object = &vm->memory_error_object;
 
     njs_lvlhsh_init(&object->hash);
     njs_lvlhsh_init(&object->shared_hash);
-    object->__proto__ = &prototypes[NJS_OBJ_TYPE_INTERNAL_ERROR].object;
+    object->__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_INTERNAL_ERROR);
     object->slots = NULL;
     object->type = NJS_OBJECT;
     object->shared = 1;
@@ -555,7 +553,7 @@ njs_memory_error_prototype_create(njs_vm
 
     function = njs_function(value);
     proto = njs_property_prototype_create(vm, &function->object.hash,
-                                          &vm->prototypes[index].object);
+                                          njs_vm_proto(vm, index));
     if (proto == NULL) {
         proto = &njs_value_undefined;
     }
diff -r 3386684745b7 -r 167f75576f49 src/njs_extern.c
--- a/src/njs_extern.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_extern.c	Mon Jul 03 12:49:00 2023 -0700
@@ -315,6 +315,77 @@ njs_vm_external_prototype(njs_vm_t *vm, 
 }
 
 
+static njs_int_t
+njs_vm_external_constructor_handler(njs_vm_t *vm, njs_object_prop_t *prop,
+    njs_value_t *value, njs_value_t *setval, njs_value_t *retval)
+{
+    njs_set_function(retval, &njs_vm_ctor(vm, njs_prop_magic32(prop)));
+
+    return NJS_OK;
+}
+
+
+njs_int_t
+njs_vm_external_constructor(njs_vm_t *vm, const njs_str_t *name,
+    njs_function_native_t native, const njs_external_t *ctor_props,
+    njs_uint_t ctor_nprops, const njs_external_t *proto_props,
+    njs_uint_t proto_nprops)
+{
+    njs_int_t               ret, index, proto_id;
+    njs_arr_t               **pprotos;
+    njs_function_t          *constructor;
+    njs_exotic_slots_t      *slots;
+    njs_object_prototype_t  *prototype;
+
+    index = njs_vm_ctor_push(vm);
+    if (njs_slow_path(index < 0)) {
+        njs_internal_error(vm, "njs_vm_ctor_push() failed");
+        return -1;
+    }
+
+    proto_id = njs_vm_external_prototype(vm, proto_props, proto_nprops);
+    if (njs_slow_path(proto_id < 0)) {
+        njs_internal_error(vm, "njs_vm_external_prototype(proto_props) failed");
+        return -1;
+    }
+
+    prototype = njs_shared_prototype(vm->shared, index);
+    njs_memzero(prototype, sizeof(njs_object_prototype_t));
+    prototype->object.type = NJS_OBJECT;
+    prototype->object.extensible = 1;
+
+    pprotos = njs_arr_item(vm->protos, proto_id);
+    slots = (*pprotos)->start;
+    prototype->object.shared_hash = slots->external_shared_hash;
+
+    proto_id = njs_vm_external_prototype(vm, ctor_props, ctor_nprops);
+    if (njs_slow_path(proto_id < 0)) {
+        njs_internal_error(vm, "njs_vm_external_prototype(ctor_props) failed");
+        return -1;
+    }
+
+    constructor = njs_shared_ctor(vm->shared, index);
+    njs_memzero(constructor, sizeof(njs_function_t));
+    constructor->object.type = NJS_FUNCTION;
+    constructor->u.native = native;
+    constructor->magic8 = index;
+    constructor->native = 1;
+    constructor->ctor = 1;
+
+    pprotos = njs_arr_item(vm->protos, proto_id);
+    slots = (*pprotos)->start;
+    constructor->object.shared_hash = slots->external_shared_hash;
+
+    ret = njs_vm_bind_handler(vm, name, njs_vm_external_constructor_handler, 0,
+                              index, 1);
+    if (njs_slow_path(ret != NJS_OK)) {
+        return NJS_ERROR;
+    }
+
+    return index;
+}
+
+
 njs_int_t
 njs_vm_external_create(njs_vm_t *vm, njs_value_t *value, njs_int_t proto_id,
     njs_external_ptr_t external, njs_bool_t shared)
diff -r 3386684745b7 -r 167f75576f49 src/njs_function.c
--- a/src/njs_function.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_function.c	Mon Jul 03 12:49:00 2023 -0700
@@ -46,10 +46,10 @@ njs_function_alloc(njs_vm_t *vm, njs_fun
     }
 
     if (async) {
-        proto = &vm->prototypes[NJS_OBJ_TYPE_ASYNC_FUNCTION].object;
+        proto = njs_vm_proto(vm, NJS_OBJ_TYPE_ASYNC_FUNCTION);
 
     } else {
-        proto = &vm->prototypes[NJS_OBJ_TYPE_FUNCTION].object;
+        proto = njs_vm_proto(vm, NJS_OBJ_TYPE_FUNCTION);
     }
 
     function->object.__proto__ = proto;
@@ -84,7 +84,7 @@ njs_vm_function_alloc(njs_vm_t *vm, njs_
     function->object.shared = shared;
     function->u.native = native;
     function->object.shared_hash = vm->shared->function_instance_hash;
-    function->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_FUNCTION].object;
+    function->object.__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_FUNCTION);
     function->object.type = NJS_FUNCTION;
 
     return function;
@@ -212,7 +212,7 @@ njs_function_copy(njs_vm_t *vm, njs_func
 
     type = njs_function_object_type(vm, function);
 
-    copy->object.__proto__ = &vm->prototypes[type].object;
+    copy->object.__proto__ = njs_vm_proto(vm, type);
     copy->object.shared = 0;
 
     if (copy->ctor) {
@@ -1370,7 +1370,7 @@ njs_function_prototype_bind(njs_vm_t *vm
     /* Bound functions have no "prototype" property. */
     function->object.shared_hash = vm->shared->arrow_instance_hash;
 
-    function->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_FUNCTION].object;
+    function->object.__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_FUNCTION);
     function->object.shared = 0;
 
     function->context = njs_function(&args[0]);
diff -r 3386684745b7 -r 167f75576f49 src/njs_json.c
--- a/src/njs_json.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_json.c	Mon Jul 03 12:49:00 2023 -0700
@@ -1979,7 +1979,7 @@ njs_vm_value_dump(njs_vm_t *vm, njs_str_
 
     if (njs_slow_path(vm->top_frame == NULL)) {
         /* An exception was thrown during compilation. */
-        njs_vm_init(vm);
+        njs_vm_runtime_init(vm);
     }
 
     if (njs_is_valid(&vm->exception)) {
diff -r 3386684745b7 -r 167f75576f49 src/njs_object.c
--- a/src/njs_object.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_object.c	Mon Jul 03 12:49:00 2023 -0700
@@ -45,7 +45,7 @@ njs_object_alloc(njs_vm_t *vm)
     if (njs_fast_path(object != NULL)) {
         njs_lvlhsh_init(&object->hash);
         njs_lvlhsh_init(&object->shared_hash);
-        object->__proto__ = &vm->prototypes[NJS_OBJ_TYPE_OBJECT].object;
+        object->__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_OBJECT);
         object->slots = NULL;
         object->type = NJS_OBJECT;
         object->shared = 0;
@@ -80,7 +80,7 @@ njs_object_value_copy(njs_vm_t *vm, njs_
 
     if (njs_fast_path(object != NULL)) {
         memcpy(object, njs_object(value), size);
-        object->__proto__ = &vm->prototypes[NJS_OBJ_TYPE_OBJECT].object;
+        object->__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_OBJECT);
         object->shared = 0;
         value->data.u.object = object;
         return object;
@@ -119,7 +119,7 @@ njs_object_value_alloc(njs_vm_t *vm, njs
     ov->object.error_data = 0;
     ov->object.fast_array = 0;
 
-    ov->object.__proto__ = &vm->prototypes[prototype_index].object;
+    ov->object.__proto__ = njs_vm_proto(vm, prototype_index);
     ov->object.slots = NULL;
 
     if (value != NULL) {
@@ -1501,7 +1501,7 @@ njs_object_get_prototype_of(njs_vm_t *vm
         index = njs_primitive_prototype_index(value->type);
 
         if (njs_is_symbol(value)) {
-            njs_set_object(retval, &vm->prototypes[index].object);
+            njs_set_object(retval, njs_vm_proto(vm, index));
 
         } else {
             njs_set_object_value(retval,
@@ -1843,7 +1843,7 @@ njs_primitive_prototype_get_proto(njs_vm
 
     } else {
         index = njs_primitive_prototype_index(value->type);
-        proto = &vm->prototypes[index].object;
+        proto = njs_vm_proto(vm, index);
     }
 
     if (proto != NULL) {
@@ -1875,7 +1875,7 @@ njs_object_prototype_create(njs_vm_t *vm
     function = njs_function(value);
     index = function - vm->constructors;
 
-    if (index >= 0 && index < NJS_OBJ_TYPE_MAX) {
+    if (index >= 0 && (size_t) index < vm->constructors_size) {
         proto = njs_property_prototype_create(vm, &function->object.hash,
                                               &vm->prototypes[index].object);
     }
@@ -2111,7 +2111,7 @@ njs_object_prototype_create_constructor(
             prototype = (njs_object_prototype_t *) object;
             index = prototype - vm->prototypes;
 
-            if (index >= 0 && index < NJS_OBJ_TYPE_MAX) {
+            if (index >= 0 && (size_t) index < vm->constructors_size) {
                 goto found;
             }
 
@@ -2128,7 +2128,7 @@ njs_object_prototype_create_constructor(
 
 found:
 
-    njs_set_function(&constructor, &vm->constructors[index]);
+    njs_set_function(&constructor, &njs_vm_ctor(vm, index));
     setval = &constructor;
 
     cons = njs_property_constructor_set(vm, &prototype->object.hash, setval);
diff -r 3386684745b7 -r 167f75576f49 src/njs_parser.c
--- a/src/njs_parser.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_parser.c	Mon Jul 03 12:49:00 2023 -0700
@@ -9199,6 +9199,10 @@ njs_parser_error(njs_vm_t *vm, njs_objec
     static const njs_value_t  file_name = njs_string("fileName");
     static const njs_value_t  line_number = njs_string("lineNumber");
 
+    if (njs_slow_path(vm->top_frame == NULL)) {
+        njs_vm_runtime_init(vm);
+    }
+
     p = msg;
     end = msg + NJS_MAX_ERROR_STR;
 
@@ -9217,7 +9221,7 @@ njs_parser_error(njs_vm_t *vm, njs_objec
         p = njs_sprintf(p, end, " in %uD", line);
     }
 
-    njs_error_new(vm, &error, &vm->prototypes[type].object, msg, p - msg);
+    njs_error_new(vm, &error, njs_vm_proto(vm, type), msg, p - msg);
 
     njs_set_number(&value, line);
     njs_value_property_set(vm, &error, njs_value_arg(&line_number), &value);
diff -r 3386684745b7 -r 167f75576f49 src/njs_promise.c
--- a/src/njs_promise.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_promise.c	Mon Jul 03 12:49:00 2023 -0700
@@ -131,7 +131,7 @@ njs_promise_alloc(njs_vm_t *vm)
     promise->object.extensible = 1;
     promise->object.error_data = 0;
     promise->object.fast_array = 0;
-    promise->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_PROMISE].object;
+    promise->object.__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_PROMISE);
     promise->object.slots = NULL;
 
     data = (njs_promise_data_t *) ((uint8_t *) promise + sizeof(njs_promise_t));
@@ -265,7 +265,7 @@ njs_promise_create_function(njs_vm_t *vm
         context = NULL;
     }
 
-    function->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_FUNCTION].object;
+    function->object.__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_FUNCTION);
     function->object.shared_hash = vm->shared->arrow_instance_hash;
     function->object.type = NJS_FUNCTION;
     function->object.extensible = 1;
@@ -1352,8 +1352,8 @@ njs_promise_perform_all(njs_vm_t *vm, nj
 
         if (handler == njs_promise_perform_any_handler) {
             error = njs_error_alloc(vm,
-                           &vm->prototypes[NJS_OBJ_TYPE_AGGREGATE_ERROR].object,
-                           NULL, &string_any_rejected, &argument);
+                                njs_vm_proto(vm, NJS_OBJ_TYPE_AGGREGATE_ERROR),
+                                NULL, &string_any_rejected, &argument);
             if (njs_slow_path(error == NULL)) {
                 return NJS_ERROR;
             }
@@ -1730,8 +1730,8 @@ njs_promise_any_reject_element_functions
         njs_mp_free(vm->mem_pool, context->remaining_elements);
 
         error = njs_error_alloc(vm,
-                          &vm->prototypes[NJS_OBJ_TYPE_AGGREGATE_ERROR].object,
-                          NULL, &string_any_rejected, &arr_value);
+                                njs_vm_proto(vm, NJS_OBJ_TYPE_AGGREGATE_ERROR),
+                                NULL, &string_any_rejected, &arr_value);
         if (njs_slow_path(error == NULL)) {
             return NJS_ERROR;
         }
diff -r 3386684745b7 -r 167f75576f49 src/njs_regexp.c
--- a/src/njs_regexp.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_regexp.c	Mon Jul 03 12:49:00 2023 -0700
@@ -503,7 +503,7 @@ njs_regexp_alloc(njs_vm_t *vm, njs_regex
     if (njs_fast_path(regexp != NULL)) {
         njs_lvlhsh_init(&regexp->object.hash);
         regexp->object.shared_hash = vm->shared->regexp_instance_hash;
-        regexp->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_REGEXP].object;
+        regexp->object.__proto__ = njs_vm_proto(vm, NJS_OBJ_TYPE_REGEXP);
         regexp->object.slots = NULL;
         regexp->object.type = NJS_REGEXP;
         regexp->object.shared = 0;
@@ -628,7 +628,7 @@ njs_regexp_prototype_flag(njs_vm_t *vm, 
     }
 
     if (njs_slow_path(!njs_is_regexp(this))) {
-        if (njs_object(this) == &vm->prototypes[NJS_OBJ_TYPE_REGEXP].object) {
+        if (njs_object(this) == njs_vm_proto(vm, NJS_OBJ_TYPE_REGEXP)) {
             njs_set_undefined(retval);
             return NJS_OK;
         }
@@ -679,7 +679,7 @@ njs_regexp_prototype_source(njs_vm_t *vm
     }
 
     if (njs_slow_path(!njs_is_regexp(this))) {
-        if (njs_object(this) == &vm->prototypes[NJS_OBJ_TYPE_REGEXP].object) {
+        if (njs_object(this) == njs_vm_proto(vm, NJS_OBJ_TYPE_REGEXP)) {
             njs_value_assign(retval, &njs_string_empty_regexp);
             return NJS_OK;
         }
@@ -1553,7 +1553,7 @@ njs_regexp_prototype_symbol_split(njs_vm
         return ret;
     }
 
-    njs_set_function(&constructor, &vm->constructors[NJS_OBJ_TYPE_REGEXP]);
+    njs_set_function(&constructor, &njs_vm_ctor(vm, NJS_OBJ_TYPE_REGEXP));
 
     ret = njs_value_species_constructor(vm, rx, &constructor, &constructor);
     if (njs_slow_path(ret != NJS_OK)) {
diff -r 3386684745b7 -r 167f75576f49 src/njs_typed_array.c
--- a/src/njs_typed_array.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_typed_array.c	Mon Jul 03 12:49:00 2023 -0700
@@ -180,7 +180,7 @@ njs_typed_array_alloc(njs_vm_t *vm, njs_
 
     njs_lvlhsh_init(&array->object.hash);
     njs_lvlhsh_init(&array->object.shared_hash);
-    array->object.__proto__ = &vm->prototypes[type].object;
+    array->object.__proto__ = njs_vm_proto(vm, type);
     array->object.type = NJS_TYPED_ARRAY;
     array->object.extensible = 1;
     array->object.fast_array = 1;
@@ -264,7 +264,7 @@ njs_typed_array_species_create(njs_vm_t 
 
     array = njs_typed_array(exemplar);
 
-    njs_set_function(&constructor, &vm->constructors[array->type]);
+    njs_set_function(&constructor, &njs_vm_ctor(vm, array->type));
 
     ret = njs_value_species_constructor(vm, exemplar, &constructor,
                                         &constructor);
@@ -2413,7 +2413,7 @@ njs_data_view_constructor(njs_vm_t *vm, 
 
     njs_lvlhsh_init(&view->object.hash);
     njs_lvlhsh_init(&view->object.shared_hash);
-    view->object.__proto__ = &vm->prototypes[view->type].object;
+    view->object.__proto__ = njs_vm_proto(vm, view->type);
     view->object.type = NJS_DATA_VIEW;
     view->object.extensible = 1;
 
diff -r 3386684745b7 -r 167f75576f49 src/njs_value.c
--- a/src/njs_value.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_value.c	Mon Jul 03 12:49:00 2023 -0700
@@ -638,7 +638,7 @@ njs_property_query(njs_vm_t *vm, njs_pro
     case NJS_NUMBER:
     case NJS_SYMBOL:
         index = njs_primitive_prototype_index(value->type);
-        obj = &vm->prototypes[index].object;
+        obj = njs_vm_proto(vm, index);
         break;
 
     case NJS_STRING:
diff -r 3386684745b7 -r 167f75576f49 src/njs_value.h
--- a/src/njs_value.h	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_value.h	Mon Jul 03 12:49:00 2023 -0700
@@ -625,9 +625,8 @@ typedef struct {
     ((value)->type >= NJS_OBJECT)
 
 
-#define njs_has_prototype(vm, value, proto)                                   \
-    (((njs_object_prototype_t *)                                              \
-        njs_object(value)->__proto__ - (vm)->prototypes) == proto)
+#define njs_has_prototype(vm, value, proto_id)                                \
+    (njs_object(value)->__proto__ == njs_vm_proto(vm, proto_id))
 
 
 #define njs_is_object_value(value)                                            \
diff -r 3386684745b7 -r 167f75576f49 src/njs_vm.c
--- a/src/njs_vm.c	Fri Jun 30 21:02:44 2023 -0700
+++ b/src/njs_vm.c	Mon Jul 03 12:49:00 2023 -0700
@@ -9,6 +9,7 @@
 
 
 static njs_int_t njs_vm_handle_events(njs_vm_t *vm);
+static njs_int_t njs_vm_protos_init(njs_vm_t *vm, njs_value_t *global);
 
 
 const njs_str_t  njs_entry_empty =          njs_str("");
@@ -78,13 +79,47 @@ njs_vm_create(njs_vm_opt_t *options)
     vm->trace.data = vm;
 
     if (options->init) {
-        ret = njs_vm_init(vm);
+        ret = njs_vm_runtime_init(vm);
         if (njs_slow_path(ret != NJS_OK)) {
             return NULL;
         }
     }
 
     for (i = 0; njs_modules[i] != NULL; i++) {
+        if (njs_modules[i]->preinit == NULL) {
+            continue;
+        }
+
+        ret = njs_modules[i]->preinit(vm);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return NULL;
+        }
+    }
+
+    if (options->addons != NULL) {
+        addons = options->addons;
+        for (i = 0; addons[i] != NULL; i++) {
+            if (addons[i]->preinit == NULL) {
+                continue;
+            }
+
+            ret = addons[i]->preinit(vm);
+            if (njs_slow_path(ret != NJS_OK)) {
+                return NULL;
+            }
+        }
+    }
+
+    ret = njs_vm_protos_init(vm, &vm->global_value);
+    if (njs_slow_path(ret != NJS_OK)) {
+        return NULL;
+    }
+
+    for (i = 0; njs_modules[i] != NULL; i++) {
+        if (njs_modules[i]->init == NULL) {
+            continue;
+        }
+
         ret = njs_modules[i]->init(vm);
         if (njs_slow_path(ret != NJS_OK)) {
             return NULL;
@@ -94,6 +129,10 @@ njs_vm_create(njs_vm_opt_t *options)
     if (options->addons != NULL) {
         addons = options->addons;
         for (i = 0; addons[i] != NULL; i++) {
+            if (addons[i]->init == NULL) {
+                continue;
+            }
+
             ret = addons[i]->init(vm);
             if (njs_slow_path(ret != NJS_OK)) {
                 return NULL;
@@ -111,6 +150,51 @@ njs_vm_create(njs_vm_opt_t *options)
 }
 
 
+njs_int_t
+njs_vm_ctor_push(njs_vm_t *vm)
+{
+    njs_function_t          *ctor;
+    njs_vm_shared_t         *shared;
+    njs_object_prototype_t  *prototype;
+
+    shared = vm->shared;

From xeioex at nginx.com  Mon Jul  3 20:34:14 2023
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Mon, 03 Jul 2023 20:34:14 +0000
Subject: [njs] Modules: introduced js_shared_dict_zone directive.
Message-ID: <hg.ff7eb3c4bf76.1688416454.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/ff7eb3c4bf76
branches:  
changeset: 2176:ff7eb3c4bf76
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Mon Jul 03 13:32:41 2023 -0700
description:
Modules: introduced js_shared_dict_zone directive.

The directive allows to declare a dictionary that is shared among the
working processes. A dictionary expects strings as keys. It stores
string or number as values. The value type is declared using
type= argument of the directive. The default type is string.

example.conf:
    # Declares a shared dictionary of strings of size 1 Mb that
    # removes key-value after 60 seconds of inactivity.
    js_shared_dict_zone zone=foo:1M timeout=60s;

    # Declares a shared dictionary of strings of size 512Kb that
    # forcibly remove oldest key-value pairs when memory is not enough.
    js_shared_dict_zone zone=bar:512K timeout=30s evict;

    # Declares a permanent number shared dictionary of size 32Kb.
    js_shared_dict_zone zone=num:32k type=number;

example.js:
    function get(r) {
        r.return(200, ngx.shared.foo.get(r.args.key));
    }

    function set(r) {
        r.return(200, ngx.shared.foo.set(r.args.key, r.args.value));
    }

    function delete(r) {
        r.return(200, ngx.shared.bar.delete(r.args.key));
    }

    function increment(r) {
        r.return(200, ngx.shared.num.incr(r.args.key, 2));
    }

In collaboration with Artem S. Povalyukhin, Jakub Jirutka and
洪志道 (Hong Zhi Dao).

This closes #437 issue on Github.

diffstat:

 nginx/config                    |     6 +-
 nginx/ngx_http_js_module.c      |    48 +-
 nginx/ngx_js.c                  |    34 +
 nginx/ngx_js.h                  |    18 +
 nginx/ngx_js_shared_dict.c      |  1586 +++++++++++++++++++++++++++++++++++++++
 nginx/ngx_js_shared_dict.h      |    19 +
 nginx/ngx_stream_js_module.c    |    48 +-
 nginx/t/js_shared_dict.t        |   299 +++++++
 nginx/t/stream_js_shared_dict.t |   188 ++++
 ts/ngx_core.d.ts                |   150 +++
 10 files changed, 2390 insertions(+), 6 deletions(-)

diffs (truncated from 2634 to 1000 lines):

diff -r 167f75576f49 -r ff7eb3c4bf76 nginx/config
--- a/nginx/config	Mon Jul 03 12:49:00 2023 -0700
+++ b/nginx/config	Mon Jul 03 13:32:41 2023 -0700
@@ -5,10 +5,12 @@ NJS_LIBXSLT=${NJS_LIBXSLT:-YES}
 NJS_ZLIB=${NJS_ZLIB:-YES}
 
 NJS_DEPS="$ngx_addon_dir/ngx_js.h \
-    $ngx_addon_dir/ngx_js_fetch.h"
+    $ngx_addon_dir/ngx_js_fetch.h \
+    $ngx_addon_dir/ngx_js_shared_dict.h"
 NJS_SRCS="$ngx_addon_dir/ngx_js.c \
     $ngx_addon_dir/ngx_js_fetch.c \
-    $ngx_addon_dir/ngx_js_regex.c"
+    $ngx_addon_dir/ngx_js_regex.c \
+    $ngx_addon_dir/ngx_js_shared_dict.c"
 
 NJS_OPENSSL_LIB=
 NJS_XSLT_LIB=
diff -r 167f75576f49 -r ff7eb3c4bf76 nginx/ngx_http_js_module.c
--- a/nginx/ngx_http_js_module.c	Mon Jul 03 12:49:00 2023 -0700
+++ b/nginx/ngx_http_js_module.c	Mon Jul 03 13:32:41 2023 -0700
@@ -252,10 +252,13 @@ static char *ngx_http_js_set(ngx_conf_t 
 static char *ngx_http_js_var(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
 static char *ngx_http_js_content(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+static char *ngx_http_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd,
+    void *conf);
 static char *ngx_http_js_body_filter_set(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
 static ngx_int_t ngx_http_js_init_conf_vm(ngx_conf_t *cf,
     ngx_js_loc_conf_t *conf);
+static void *ngx_http_js_create_main_conf(ngx_conf_t *cf);
 static void *ngx_http_js_create_loc_conf(ngx_conf_t *cf);
 static char *ngx_http_js_merge_loc_conf(ngx_conf_t *cf, void *parent,
     void *child);
@@ -393,6 +396,13 @@ static ngx_command_t  ngx_http_js_comman
 
 #endif
 
+    { ngx_string("js_shared_dict_zone"),
+      NGX_HTTP_MAIN_CONF|NGX_CONF_1MORE,
+      ngx_http_js_shared_dict_zone,
+      0,
+      0,
+      NULL },
+
       ngx_null_command
 };
 
@@ -401,7 +411,7 @@ static ngx_http_module_t  ngx_http_js_mo
     NULL,                          /* preconfiguration */
     ngx_http_js_init,              /* postconfiguration */
 
-    NULL,                          /* create main configuration */
+    ngx_http_js_create_main_conf,  /* create main configuration */
     NULL,                          /* init main configuration */
 
     NULL,                          /* create server configuration */
@@ -775,6 +785,7 @@ static uintptr_t ngx_http_js_uptr[] = {
     (uintptr_t) ngx_http_js_fetch_timeout,
     (uintptr_t) ngx_http_js_buffer_size,
     (uintptr_t) ngx_http_js_max_response_buffer_size,
+    (uintptr_t) 0 /* main_conf ptr */,
 };
 
 
@@ -798,6 +809,7 @@ njs_module_t *njs_http_js_addon_modules[
      */
     &ngx_js_ngx_module,
     &ngx_js_fetch_module,
+    &ngx_js_shared_dict_module,
 #ifdef NJS_HAVE_OPENSSL
     &njs_webcrypto_module,
 #endif
@@ -4149,10 +4161,14 @@ ngx_js_http_init(njs_vm_t *vm)
 static ngx_int_t
 ngx_http_js_init_conf_vm(ngx_conf_t *cf, ngx_js_loc_conf_t *conf)
 {
-    njs_vm_opt_t  options;
+    njs_vm_opt_t         options;
+    ngx_js_main_conf_t  *jmcf;
 
     njs_vm_opt_init(&options);
 
+    jmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_js_module);
+    ngx_http_js_uptr[NGX_JS_MAIN_CONF_INDEX] = (uintptr_t) jmcf;
+
     options.backtrace = 1;
     options.unhandled_rejection = NJS_VM_OPT_UNHANDLED_REJECTION_THROW;
     options.ops = &ngx_http_js_ops;
@@ -4293,6 +4309,14 @@ ngx_http_js_content(ngx_conf_t *cf, ngx_
 
 
 static char *
+ngx_http_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd,
+    void *conf)
+{
+    return ngx_js_shared_dict_zone(cf, cmd, conf, &ngx_http_js_module);
+}
+
+
+static char *
 ngx_http_js_body_filter_set(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
 {
     ngx_http_js_loc_conf_t *jlcf = conf;
@@ -4331,6 +4355,26 @@ ngx_http_js_body_filter_set(ngx_conf_t *
 
 
 static void *
+ngx_http_js_create_main_conf(ngx_conf_t *cf)
+{
+    ngx_js_main_conf_t  *jmcf;
+
+    jmcf = ngx_pcalloc(cf->pool, sizeof(ngx_js_main_conf_t));
+    if (jmcf == NULL) {
+        return NULL;
+    }
+
+    /*
+     * set by ngx_pcalloc():
+     *
+     *     jmcf->dicts = NULL;
+     */
+
+    return jmcf;
+}
+
+
+static void *
 ngx_http_js_create_loc_conf(ngx_conf_t *cf)
 {
     ngx_http_js_loc_conf_t  *conf =
diff -r 167f75576f49 -r ff7eb3c4bf76 nginx/ngx_js.c
--- a/nginx/ngx_js.c	Mon Jul 03 12:49:00 2023 -0700
+++ b/nginx/ngx_js.c	Mon Jul 03 13:32:41 2023 -0700
@@ -32,6 +32,28 @@ static void ngx_js_cleanup_vm(void *data
 static njs_int_t ngx_js_core_init(njs_vm_t *vm);
 
 
+static njs_external_t  ngx_js_ext_global_shared[] = {
+
+    {
+        .flags = NJS_EXTERN_PROPERTY | NJS_EXTERN_SYMBOL,
+        .name.symbol = NJS_SYMBOL_TO_STRING_TAG,
+        .u.property = {
+            .value = "GlobalShared",
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_SELF,
+        .u.object = {
+            .enumerable = 1,
+            .prop_handler = njs_js_ext_global_shared_prop,
+            .keys = njs_js_ext_global_shared_keys,
+        }
+    },
+
+};
+
+
 static njs_external_t  ngx_js_ext_core[] = {
 
     {
@@ -113,6 +135,18 @@ static njs_external_t  ngx_js_ext_core[]
     },
 
     {
+        .flags = NJS_EXTERN_OBJECT,
+        .name.string = njs_str("shared"),
+        .enumerable = 1,
+        .writable = 1,
+        .u.object = {
+            .enumerable = 1,
+            .properties = ngx_js_ext_global_shared,
+            .nproperties = njs_nitems(ngx_js_ext_global_shared),
+        }
+    },
+
+    {
         .flags = NJS_EXTERN_PROPERTY,
         .name.string = njs_str("prefix"),
         .enumerable = 1,
diff -r 167f75576f49 -r ff7eb3c4bf76 nginx/ngx_js.h
--- a/nginx/ngx_js.h	Mon Jul 03 12:49:00 2023 -0700
+++ b/nginx/ngx_js.h	Mon Jul 03 13:32:41 2023 -0700
@@ -14,6 +14,7 @@
 #include <ngx_core.h>
 #include <njs.h>
 #include "ngx_js_fetch.h"
+#include "ngx_js_shared_dict.h"
 
 
 #define NGX_JS_UNSET        0
@@ -43,6 +44,9 @@ typedef ngx_flag_t (*ngx_external_size_p
     njs_external_ptr_t e);
 typedef ngx_ssl_t *(*ngx_external_ssl_pt)(njs_vm_t *vm, njs_external_ptr_t e);
 
+
+typedef struct ngx_js_dict_s  ngx_js_dict_t;
+
 typedef struct {
     ngx_str_t              name;
     ngx_str_t              path;
@@ -51,6 +55,10 @@ typedef struct {
 } ngx_js_named_path_t;
 
 
+#define NGX_JS_COMMON_MAIN_CONF                                               \
+    ngx_js_dict_t         *dicts                                              \
+
+
 #define _NGX_JS_COMMON_LOC_CONF                                               \
     njs_vm_t              *vm;                                                \
     ngx_array_t           *imports;                                           \
@@ -81,6 +89,11 @@ typedef struct {
 
 
 typedef struct {
+    NGX_JS_COMMON_MAIN_CONF;
+} ngx_js_main_conf_t;
+
+
+typedef struct {
     NGX_JS_COMMON_LOC_CONF;
 } ngx_js_loc_conf_t;
 
@@ -105,6 +118,9 @@ typedef struct {
     ((ngx_external_size_pt) njs_vm_meta(vm, 8))(vm, e)
 #define ngx_external_max_response_buffer_size(vm, e)                          \
     ((ngx_external_size_pt) njs_vm_meta(vm, 9))(vm, e)
+#define NGX_JS_MAIN_CONF_INDEX  10
+#define ngx_main_conf(vm)                                                     \
+	((ngx_js_main_conf_t *) njs_vm_meta(vm, NGX_JS_MAIN_CONF_INDEX))
 
 
 #define ngx_js_prop(vm, type, value, start, len)                              \
@@ -134,6 +150,8 @@ ngx_int_t ngx_js_init_conf_vm(ngx_conf_t
 ngx_js_loc_conf_t *ngx_js_create_conf(ngx_conf_t *cf, size_t size);
 char * ngx_js_merge_conf(ngx_conf_t *cf, void *parent, void *child,
    ngx_int_t (*init_vm)(ngx_conf_t *cf, ngx_js_loc_conf_t *conf));
+char *ngx_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf,
+    void *tag);
 
 njs_int_t ngx_js_ext_string(njs_vm_t *vm, njs_object_prop_t *prop,
     njs_value_t *value, njs_value_t *setval, njs_value_t *retval);
diff -r 167f75576f49 -r ff7eb3c4bf76 nginx/ngx_js_shared_dict.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/nginx/ngx_js_shared_dict.c	Mon Jul 03 13:32:41 2023 -0700
@@ -0,0 +1,1586 @@
+
+/*
+ * Copyright (C) Dmitry Volyntsev
+ * Copyright (C) NGINX, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include "ngx_js.h"
+#include "ngx_js_shared_dict.h"
+
+
+typedef struct {
+    ngx_rbtree_t           rbtree;
+    ngx_rbtree_node_t      sentinel;
+    ngx_atomic_t           rwlock;
+
+    ngx_rbtree_t           rbtree_expire;
+    ngx_rbtree_node_t      sentinel_expire;
+} ngx_js_dict_sh_t;
+
+
+typedef struct {
+    ngx_str_node_t         sn;
+    ngx_rbtree_node_t      expire;
+    union {
+        ngx_str_t          value;
+        double             number;
+    } u;
+} ngx_js_dict_node_t;
+
+
+struct ngx_js_dict_s {
+    ngx_shm_zone_t        *shm_zone;
+    ngx_js_dict_sh_t      *sh;
+    ngx_slab_pool_t       *shpool;
+
+    ngx_msec_t             timeout;
+    ngx_flag_t             evict;
+#define NGX_JS_DICT_TYPE_STRING  0
+#define NGX_JS_DICT_TYPE_NUMBER  1
+    ngx_uint_t             type;
+
+    ngx_js_dict_t         *next;
+};
+
+
+static njs_int_t njs_js_ext_shared_dict_capacity(njs_vm_t *vm,
+    njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval,
+    njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_clear(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t flags, njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_delete(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused, njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_free_space(njs_vm_t *vm,
+    njs_value_t *args, njs_uint_t nargs, njs_index_t unused,
+    njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_get(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused, njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_has(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused, njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_keys(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused, njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_incr(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused, njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_name(njs_vm_t *vm,
+    njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval,
+    njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_pop(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused, njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_set(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t flags, njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_size(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused, njs_value_t *retval);
+static njs_int_t njs_js_ext_shared_dict_type(njs_vm_t *vm,
+    njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval,
+    njs_value_t *retval);
+static ngx_js_dict_node_t *ngx_js_dict_lookup(ngx_js_dict_t *dict,
+    njs_str_t *key);
+
+#define NGX_JS_DICT_FLAG_MUST_EXIST       1
+#define NGX_JS_DICT_FLAG_MUST_NOT_EXIST   2
+
+static ngx_int_t ngx_js_dict_set(njs_vm_t *vm, ngx_js_dict_t *dict,
+    njs_str_t *key, njs_value_t *value, unsigned flags);
+static ngx_int_t ngx_js_dict_add(ngx_js_dict_t *dict, njs_str_t *key,
+    njs_value_t *value, ngx_msec_t now);
+static ngx_int_t ngx_js_dict_update(ngx_js_dict_t *dict,
+    ngx_js_dict_node_t *node, njs_value_t *value, ngx_msec_t now);
+static ngx_int_t ngx_js_dict_get(njs_vm_t *vm, ngx_js_dict_t *dict,
+    njs_str_t *key, njs_value_t *retval);
+static ngx_int_t ngx_js_dict_incr(ngx_js_dict_t *dict, njs_str_t *key,
+    njs_value_t *delta, njs_value_t *init, double *value);
+static ngx_int_t ngx_js_dict_delete(njs_vm_t *vm, ngx_js_dict_t *dict,
+    njs_str_t *key, njs_value_t *retval);
+static ngx_int_t ngx_js_dict_copy_value_locked(njs_vm_t *vm,
+    ngx_js_dict_t *dict, ngx_js_dict_node_t *node, njs_value_t *retval);
+
+static void ngx_js_dict_expire(ngx_js_dict_t *dict, ngx_msec_t now);
+static void ngx_js_dict_evict(ngx_js_dict_t *dict, ngx_int_t count);
+
+static njs_int_t ngx_js_dict_shared_error_name(njs_vm_t *vm,
+    njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval,
+    njs_value_t *retval);
+
+static ngx_int_t ngx_js_dict_init_zone(ngx_shm_zone_t *shm_zone, void *data);
+static njs_int_t ngx_js_shared_dict_preinit(njs_vm_t *vm);
+static njs_int_t ngx_js_shared_dict_init(njs_vm_t *vm);
+
+
+static njs_external_t  ngx_js_ext_shared_dict[] = {
+
+    {
+        .flags = NJS_EXTERN_PROPERTY | NJS_EXTERN_SYMBOL,
+        .name.symbol = NJS_SYMBOL_TO_STRING_TAG,
+        .u.property = {
+            .value = "SharedDict",
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("add"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_set,
+            .magic8 = NGX_JS_DICT_FLAG_MUST_NOT_EXIST,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_PROPERTY,
+        .name.string = njs_str("capacity"),
+        .enumerable = 1,
+        .u.property = {
+            .handler = njs_js_ext_shared_dict_capacity,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("clear"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_clear,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("freeSpace"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_free_space,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("delete"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_delete,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("incr"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_incr,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("get"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_get,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("has"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_has,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("keys"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_keys,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_PROPERTY,
+        .name.string = njs_str("name"),
+        .enumerable = 1,
+        .u.property = {
+            .handler = njs_js_ext_shared_dict_name,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("pop"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_pop,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("replace"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_set,
+            .magic8 = NGX_JS_DICT_FLAG_MUST_EXIST,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("set"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_set,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_METHOD,
+        .name.string = njs_str("size"),
+        .writable = 1,
+        .configurable = 1,
+        .enumerable = 1,
+        .u.method = {
+            .native = njs_js_ext_shared_dict_size,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_PROPERTY,
+        .name.string = njs_str("type"),
+        .enumerable = 1,
+        .u.property = {
+            .handler = njs_js_ext_shared_dict_type,
+        }
+    },
+
+};
+
+
+static njs_external_t  ngx_js_ext_error_ctor_props[] = {
+
+    {
+        .flags = NJS_EXTERN_PROPERTY,
+        .name.string = njs_str("name"),
+        .enumerable = 1,
+        .u.property = {
+            .handler = ngx_js_dict_shared_error_name,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_PROPERTY,
+        .name.string = njs_str("prototype"),
+        .enumerable = 1,
+        .u.property = {
+            .handler = njs_object_prototype_create,
+        }
+    },
+
+};
+
+
+static njs_external_t  ngx_js_ext_error_proto_props[] = {
+
+    {
+        .flags = NJS_EXTERN_PROPERTY,
+        .name.string = njs_str("name"),
+        .enumerable = 1,
+        .u.property = {
+            .handler = ngx_js_dict_shared_error_name,
+        }
+    },
+
+    {
+        .flags = NJS_EXTERN_PROPERTY,
+        .name.string = njs_str("prototype"),
+        .enumerable = 1,
+        .u.property = {
+            .handler = njs_object_prototype_create,
+        }
+    },
+
+};
+
+
+static njs_int_t    ngx_js_shared_dict_proto_id;
+static njs_int_t    ngx_js_shared_dict_error_id;
+
+
+njs_module_t  ngx_js_shared_dict_module = {
+    .name = njs_str("shared_dict"),
+    .preinit = ngx_js_shared_dict_preinit,
+    .init = ngx_js_shared_dict_init,
+};
+
+
+njs_int_t
+njs_js_ext_global_shared_prop(njs_vm_t *vm, njs_object_prop_t *prop,
+    njs_value_t *value, njs_value_t *setval, njs_value_t *retval)
+{
+    njs_int_t            ret;
+    njs_str_t            name;
+    ngx_js_dict_t       *dict;
+    ngx_shm_zone_t      *shm_zone;
+    ngx_js_main_conf_t  *conf;
+
+    ret = njs_vm_prop_name(vm, prop, &name);
+    if (ret != NJS_OK) {
+        return NJS_ERROR;
+    }
+
+    conf = ngx_main_conf(vm);
+
+    for (dict = conf->dicts; dict != NULL; dict = dict->next) {
+        shm_zone = dict->shm_zone;
+
+        if (shm_zone->shm.name.len == name.length
+            && ngx_strncmp(shm_zone->shm.name.data, name.start,
+                           name.length)
+               == 0)
+        {
+            ret = njs_vm_external_create(vm, retval,
+                                         ngx_js_shared_dict_proto_id,
+                                         shm_zone, 0);
+            if (ret != NJS_OK) {
+                njs_vm_internal_error(vm, "sharedDict creation failed");
+                return NJS_ERROR;
+            }
+
+            return NJS_OK;
+        }
+    }
+
+    njs_value_null_set(retval);
+
+    return NJS_DECLINED;
+}
+
+
+njs_int_t
+njs_js_ext_global_shared_keys(njs_vm_t *vm, njs_value_t *unused,
+    njs_value_t *keys)
+{
+    njs_int_t            rc;
+    njs_value_t         *value;
+    ngx_js_dict_t       *dict;
+    ngx_shm_zone_t      *shm_zone;
+    ngx_js_main_conf_t  *conf;
+
+    conf = ngx_main_conf(vm);
+
+    rc = njs_vm_array_alloc(vm, keys, 4);
+    if (rc != NJS_OK) {
+        return NJS_ERROR;
+    }
+
+    for (dict = conf->dicts; dict != NULL; dict = dict->next) {
+        shm_zone = dict->shm_zone;
+
+        value = njs_vm_array_push(vm, keys);
+        if (value == NULL) {
+            return NJS_ERROR;
+        }
+
+        rc = njs_vm_value_string_set(vm, value, shm_zone->shm.name.data,
+                                     shm_zone->shm.name.len);
+        if (rc != NJS_OK) {
+            return NJS_ERROR;
+        }
+    }
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_js_ext_shared_dict_capacity(njs_vm_t *vm, njs_object_prop_t *prop,
+    njs_value_t *value, njs_value_t *setval, njs_value_t *retval)
+{
+    ngx_shm_zone_t  *shm_zone;
+
+    shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id, value);
+    if (shm_zone == NULL) {
+        njs_value_undefined_set(retval);
+        return NJS_DECLINED;
+    }
+
+    njs_value_number_set(retval, shm_zone->shm.size);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_js_ext_shared_dict_clear(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused, njs_value_t *retval)
+{
+    ngx_js_dict_t   *dict;
+    ngx_shm_zone_t  *shm_zone;
+
+    shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id,
+                               njs_argument(args, 0));
+    if (shm_zone == NULL) {
+        njs_vm_type_error(vm, "\"this\" is not a shared dict");
+        return NJS_ERROR;
+    }
+
+    dict = shm_zone->data;
+
+    ngx_rwlock_wlock(&dict->sh->rwlock);
+
+    ngx_js_dict_evict(dict, 0x7fffffff /* INT_MAX */);
+
+    ngx_rwlock_unlock(&dict->sh->rwlock);
+
+    njs_value_undefined_set(retval);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_js_ext_shared_dict_free_space(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused, njs_value_t *retval)
+{
+    size_t           bytes;
+    ngx_js_dict_t   *dict;
+    ngx_shm_zone_t  *shm_zone;
+
+    shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id,
+                               njs_argument(args, 0));
+    if (shm_zone == NULL) {
+        njs_vm_type_error(vm, "\"this\" is not a shared dict");
+        return NJS_ERROR;
+    }
+
+    dict = shm_zone->data;
+
+    ngx_rwlock_rlock(&dict->sh->rwlock);
+    bytes = dict->shpool->pfree * ngx_pagesize;
+    ngx_rwlock_unlock(&dict->sh->rwlock);
+
+    njs_value_number_set(retval, bytes);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_js_ext_shared_dict_delete(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused, njs_value_t *retval)
+{
+    ngx_int_t        rc;
+    njs_str_t        key;
+    ngx_shm_zone_t  *shm_zone;
+
+    shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id,
+                               njs_argument(args, 0));
+    if (shm_zone == NULL) {
+        njs_vm_type_error(vm, "\"this\" is not a shared dict");
+        return NJS_ERROR;
+    }
+
+    if (ngx_js_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) {
+        return NJS_ERROR;
+    }
+
+    rc = ngx_js_dict_delete(vm, shm_zone->data, &key, NULL);
+
+    njs_value_boolean_set(retval, rc == NGX_OK);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_js_ext_shared_dict_get(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused, njs_value_t *retval)
+{
+    ngx_int_t        rc;
+    njs_str_t        key;
+    ngx_shm_zone_t  *shm_zone;
+
+    shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id,
+                               njs_argument(args, 0));
+    if (shm_zone == NULL) {
+        njs_vm_type_error(vm, "\"this\" is not a shared dict");
+        return NJS_ERROR;
+    }
+
+    if (ngx_js_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) {
+        return NJS_ERROR;
+    }
+
+    rc = ngx_js_dict_get(vm, shm_zone->data, &key, retval);
+    if (njs_slow_path(rc == NGX_ERROR)) {
+        njs_vm_error(vm, "failed to get value from shared dict");
+        return NJS_ERROR;
+    }
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_js_ext_shared_dict_has(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused, njs_value_t *retval)
+{
+    njs_str_t            key;
+    ngx_msec_t           now;
+    ngx_time_t          *tp;
+    ngx_js_dict_t       *dict;
+    ngx_shm_zone_t      *shm_zone;
+    ngx_js_dict_node_t  *node;
+
+    shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id,
+                               njs_argument(args, 0));
+    if (shm_zone == NULL) {
+        njs_vm_type_error(vm, "\"this\" is not a shared dict");
+        return NJS_ERROR;
+    }
+
+    if (ngx_js_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) {
+        return NJS_ERROR;
+    }
+
+    dict = shm_zone->data;
+
+    ngx_rwlock_rlock(&dict->sh->rwlock);
+
+    node = ngx_js_dict_lookup(dict, &key);
+
+    if (node != NULL && dict->timeout) {
+        tp = ngx_timeofday();
+        now = tp->sec * 1000 + tp->msec;
+
+        if (now >= node->expire.key) {
+            node = NULL;
+        }
+    }
+
+    ngx_rwlock_unlock(&dict->sh->rwlock);
+
+    njs_value_boolean_set(retval, node != NULL);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_js_ext_shared_dict_keys(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused, njs_value_t *retval)
+{
+    njs_int_t            rc;
+    ngx_int_t            max_count;
+    njs_value_t         *value;
+    ngx_rbtree_t        *rbtree;
+    ngx_js_dict_t       *dict;
+    ngx_shm_zone_t      *shm_zone;
+    ngx_rbtree_node_t   *rn;
+    ngx_js_dict_node_t  *node;
+
+    shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id,
+                               njs_argument(args, 0));
+    if (shm_zone == NULL) {
+        njs_vm_type_error(vm, "\"this\" is not a shared dict");
+        return NJS_ERROR;
+    }
+
+    dict = shm_zone->data;
+
+    max_count = 1024;
+
+    if (nargs > 1) {
+        if (ngx_js_integer(vm, njs_arg(args, nargs, 1), &max_count) != NGX_OK) {
+            return NJS_ERROR;
+        }
+    }
+
+    rc = njs_vm_array_alloc(vm, retval, 8);
+    if (rc != NJS_OK) {
+        return NJS_ERROR;
+    }
+
+    ngx_rwlock_rlock(&dict->sh->rwlock);
+
+    rbtree = &dict->sh->rbtree;
+
+    if (rbtree->root == rbtree->sentinel) {
+        goto done;
+    }
+
+    for (rn = ngx_rbtree_min(rbtree->root, rbtree->sentinel);
+         rn != NULL;
+         rn = ngx_rbtree_next(rbtree, rn))
+    {
+        if (max_count-- == 0) {
+            break;
+        }
+
+        node = (ngx_js_dict_node_t *) rn;
+
+        value = njs_vm_array_push(vm, retval);
+        if (value == NULL) {
+            goto fail;
+        }
+
+        rc = njs_vm_value_string_set(vm, value, node->sn.str.data,
+                                     node->sn.str.len);
+        if (rc != NJS_OK) {
+            goto fail;
+        }
+    }
+
+done:
+
+    ngx_rwlock_unlock(&dict->sh->rwlock);
+
+    return NJS_OK;
+
+fail:
+
+    ngx_rwlock_unlock(&dict->sh->rwlock);
+
+    return NJS_ERROR;
+}
+
+
+static njs_int_t
+njs_js_ext_shared_dict_incr(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused, njs_value_t *retval)
+{
+    double               value;
+    ngx_int_t            rc;
+    njs_str_t            key;
+    njs_value_t         *delta, *init;
+    ngx_js_dict_t       *dict;
+    ngx_shm_zone_t      *shm_zone;
+    njs_opaque_value_t   lvalue;
+
+    shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id,
+                               njs_argument(args, 0));
+    if (shm_zone == NULL) {
+        njs_vm_type_error(vm, "\"this\" is not a shared dict");
+        return NJS_ERROR;
+    }
+
+    dict = shm_zone->data;
+
+    if (dict->type != NGX_JS_DICT_TYPE_NUMBER) {
+        njs_vm_type_error(vm, "shared dict is not a number dict");
+        return NJS_ERROR;
+    }
+
+    if (ngx_js_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) {
+        return NJS_ERROR;
+    }
+
+    delta = njs_arg(args, nargs, 2);
+    if (!njs_value_is_number(delta)) {
+        njs_vm_type_error(vm, "delta is not a number");
+        return NJS_ERROR;
+    }
+
+    init = njs_lvalue_arg(njs_value_arg(&lvalue), args, nargs, 3);
+    if (!njs_value_is_number(init) && !njs_value_is_undefined(init)) {
+        njs_vm_type_error(vm, "init value is not a number");
+        return NJS_ERROR;
+    }
+
+    if (njs_value_is_undefined(init)) {
+        njs_value_number_set(init, 0);
+    }
+
+    rc = ngx_js_dict_incr(shm_zone->data, &key, delta, init, &value);
+    if (rc == NGX_ERROR) {
+        njs_vm_error(vm, "failed to increment value in shared dict");
+        return NJS_ERROR;
+    }
+
+    njs_value_number_set(retval, value);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_js_ext_shared_dict_name(njs_vm_t *vm, njs_object_prop_t *prop,
+    njs_value_t *value, njs_value_t *setval, njs_value_t *retval)
+{
+    ngx_shm_zone_t  *shm_zone;
+
+    shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id, value);
+    if (shm_zone == NULL) {
+        njs_value_undefined_set(retval);
+        return NJS_DECLINED;
+    }
+
+    return njs_vm_value_string_set(vm, retval, shm_zone->shm.name.data,
+                                   shm_zone->shm.name.len);
+}
+
+
+static njs_int_t

From arut at nginx.com  Thu Jul  6 13:57:13 2023
From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=)
Date: Thu, 06 Jul 2023 17:57:13 +0400
Subject: [PATCH 0 of 4] QUIC path MTU discovery
In-Reply-To: <patchbomb.1680015097@arut-laptop>
References: <patchbomb.1680015097@arut-laptop>
Message-ID: <patchbomb.1688651833@arut-laptop>

Updated PMTUD:

- Eliminated getting local interface MTU.  This was too intrusive and platform-
  dependent.

- Added timeout (1 sec) before each PMTUD iteration to reduce PMTUD traffic
  in short connections.

- Size search reimplemented.  MTU is increased by 256 bytes until failure,
  after which binary search is used.  Real life MTU will be around 1500, while
  initial payload size is 1200.  Doing binary search from 65k down to 1500 will
  produce a lot of failed syscalls and useless traffic.  Ascending from 1200
  to a first failre will be quick.

--
Roman Arutyunyan

From arut at nginx.com  Thu Jul  6 13:57:14 2023
From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=)
Date: Thu, 06 Jul 2023 17:57:14 +0400
Subject: [PATCH 1 of 4] QUIC: removed path->limited flag
In-Reply-To: <patchbomb.1688651833@arut-laptop>
References: <patchbomb.1688651833@arut-laptop>
Message-ID: <df01e8582189126cd5de.1688651834@arut-laptop>

# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1688651341 -14400
#      Thu Jul 06 17:49:01 2023 +0400
# Node ID df01e8582189126cd5defd8f4bb021fec1a1a35a
# Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
QUIC: removed path->limited flag.

Its value is the opposite of path->validated.

diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
--- a/src/event/quic/ngx_event_quic.c
+++ b/src/event/quic/ngx_event_quic.c
@@ -1013,7 +1013,6 @@ ngx_quic_handle_payload(ngx_connection_t
 
         if (!qc->path->validated) {
             qc->path->validated = 1;
-            qc->path->limited = 0;
             ngx_quic_path_dbg(c, "in handshake", qc->path);
             ngx_post_event(&qc->push, &ngx_posted_events);
         }
diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h
--- a/src/event/quic/ngx_event_quic_connection.h
+++ b/src/event/quic/ngx_event_quic_connection.h
@@ -101,7 +101,6 @@ struct ngx_quic_path_s {
     u_char                            text[NGX_SOCKADDR_STRLEN];
     unsigned                          validated:1;
     unsigned                          validating:1;
-    unsigned                          limited:1;
 };
 
 
diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c
--- a/src/event/quic/ngx_event_quic_migration.c
+++ b/src/event/quic/ngx_event_quic_migration.c
@@ -168,7 +168,6 @@ valid:
 
     path->validated = 1;
     path->validating = 0;
-    path->limited = 0;
 
     ngx_quic_set_path_timer(c);
 
@@ -208,8 +207,6 @@ ngx_quic_new_path(ngx_connection_t *c,
     path->cid = cid;
     cid->used = 1;
 
-    path->limited = 1;
-
     path->seqnum = qc->path_seqnum++;
 
     path->sockaddr = &path->sa.sockaddr;
@@ -665,7 +662,6 @@ ngx_quic_path_validation_handler(ngx_eve
 
         path->validated = 0;
         path->validating = 0;
-        path->limited = 1;
 
 
         /* RFC 9000, 9.3.2.  On-Path Address Spoofing
diff --git a/src/event/quic/ngx_event_quic_migration.h b/src/event/quic/ngx_event_quic_migration.h
--- a/src/event/quic/ngx_event_quic_migration.h
+++ b/src/event/quic/ngx_event_quic_migration.h
@@ -18,11 +18,10 @@
 #define NGX_QUIC_PATH_BACKUP    2
 
 #define ngx_quic_path_dbg(c, msg, path)                                       \
-    ngx_log_debug7(NGX_LOG_DEBUG_EVENT, c->log, 0,                            \
-                   "quic path seq:%uL %s sent:%O recvd:%O state:%s%s%s",      \
+    ngx_log_debug6(NGX_LOG_DEBUG_EVENT, c->log, 0,                            \
+                   "quic path seq:%uL %s sent:%O recvd:%O state:%s%s",        \
                    path->seqnum, msg, path->sent, path->received,             \
-                   path->limited ? "L" : "", path->validated ? "V": "N",      \
-                   path->validating ? "R": "");
+                   path->validated ? "V": "N", path->validating ? "R": "");
 
 ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c,
     ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f);
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -281,7 +281,7 @@ ngx_quic_allow_segmentation(ngx_connecti
         return 0;
     }
 
-    if (qc->path->limited) {
+    if (!qc->path->validated) {
         /* don't even try to be faster on non-validated paths */
         return 0;
     }
@@ -1278,7 +1278,7 @@ ngx_quic_path_limit(ngx_connection_t *c,
 {
     off_t  max;
 
-    if (path->limited) {
+    if (!path->validated) {
         max = path->received * 3;
         max = (path->sent >= max) ? 0 : max - path->sent;
 
diff --git a/src/event/quic/ngx_event_quic_socket.c b/src/event/quic/ngx_event_quic_socket.c
--- a/src/event/quic/ngx_event_quic_socket.c
+++ b/src/event/quic/ngx_event_quic_socket.c
@@ -82,7 +82,6 @@ ngx_quic_open_sockets(ngx_connection_t *
 
     if (pkt->validated) {
         qc->path->validated = 1;
-        qc->path->limited = 0;
     }
 
     ngx_quic_path_dbg(c, "set active", qc->path);

From arut at nginx.com  Thu Jul  6 13:57:15 2023
From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=)
Date: Thu, 06 Jul 2023 17:57:15 +0400
Subject: [PATCH 2 of 4] QUIC: removed explicit packet padding for certain
 frames
In-Reply-To: <patchbomb.1688651833@arut-laptop>
References: <patchbomb.1688651833@arut-laptop>
Message-ID: <f98ff9b62d2bccac10b6.1688651835@arut-laptop>

# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1688628647 -14400
#      Thu Jul 06 11:30:47 2023 +0400
# Node ID f98ff9b62d2bccac10b603b8183eca2ff0f4183e
# Parent  df01e8582189126cd5defd8f4bb021fec1a1a35a
QUIC: removed explicit packet padding for certain frames.

The frames for which the padding is removed are PATH_CHALLENGE and
PATH_RESPONSE, which are sent separately by ngx_quic_frame_sendto().

diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -525,7 +525,7 @@ ngx_quic_output_packet(ngx_connection_t 
     ssize_t                 flen;
     ngx_str_t               res;
     ngx_int_t               rc;
-    ngx_uint_t              nframes, expand;
+    ngx_uint_t              nframes;
     ngx_msec_t              now;
     ngx_queue_t            *q;
     ngx_quic_frame_t       *f;
@@ -560,7 +560,6 @@ ngx_quic_output_packet(ngx_connection_t 
     nframes = 0;
     p = src;
     len = 0;
-    expand = 0;
 
     for (q = ngx_queue_head(&ctx->frames);
          q != ngx_queue_sentinel(&ctx->frames);
@@ -568,33 +567,6 @@ ngx_quic_output_packet(ngx_connection_t 
     {
         f = ngx_queue_data(q, ngx_quic_frame_t, queue);
 
-        if (!expand && (f->type == NGX_QUIC_FT_PATH_RESPONSE
-                        || f->type == NGX_QUIC_FT_PATH_CHALLENGE))
-        {
-            /*
-             * RFC 9000, 8.2.1.  Initiating Path Validation
-             *
-             * An endpoint MUST expand datagrams that contain a
-             * PATH_CHALLENGE frame to at least the smallest allowed
-             * maximum datagram size of 1200 bytes...
-             *
-             * (same applies to PATH_RESPONSE frames)
-             */
-
-            if (max < 1200) {
-                /* expanded packet will not fit */
-                break;
-            }
-
-            if (min < 1200) {
-                min = 1200;
-
-                min_payload = ngx_quic_payload_size(&pkt, min);
-            }
-
-            expand = 1;
-        }
-
         if (len >= max_payload) {
             break;
         }

From arut at nginx.com  Thu Jul  6 13:57:16 2023
From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=)
Date: Thu, 06 Jul 2023 17:57:16 +0400
Subject: [PATCH 3 of 4] QUIC: allowed ngx_quic_frame_sendto() to return
 NGX_AGAIN
In-Reply-To: <patchbomb.1688651833@arut-laptop>
References: <patchbomb.1688651833@arut-laptop>
Message-ID: <a1ea543d009311765144.1688651836@arut-laptop>

# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1679920727 -14400
#      Mon Mar 27 16:38:47 2023 +0400
# Node ID a1ea543d009311765144351b2557c00c8e6445bf
# Parent  f98ff9b62d2bccac10b603b8183eca2ff0f4183e
QUIC: allowed ngx_quic_frame_sendto() to return NGX_AGAIN.

Previously, NGX_AGAIN returned by ngx_send() was treated by
ngx_quic_frame_sendto() as error, which triggered errors in its callers.
However, a blocked socket is not an error.  Now NGX_AGAIN is passed as is to
the ngx_quic_frame_sendto() callers, which can safely ignore it.

diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c
--- a/src/event/quic/ngx_event_quic_migration.c
+++ b/src/event/quic/ngx_event_quic_migration.c
@@ -46,7 +46,7 @@ ngx_quic_handle_path_challenge_frame(ngx
      * An endpoint MUST expand datagrams that contain a PATH_RESPONSE frame
      * to at least the smallest allowed maximum datagram size of 1200 bytes.
      */
-    if (ngx_quic_frame_sendto(c, &frame, 1200, pkt->path) != NGX_OK) {
+    if (ngx_quic_frame_sendto(c, &frame, 1200, pkt->path) == NGX_ERROR) {
         return NGX_ERROR;
     }
 
@@ -544,13 +544,13 @@ ngx_quic_send_path_challenge(ngx_connect
      */
 
      /* same applies to PATH_RESPONSE frames */
-    if (ngx_quic_frame_sendto(c, &frame, 1200, path) != NGX_OK) {
+    if (ngx_quic_frame_sendto(c, &frame, 1200, path) == NGX_ERROR) {
         return NGX_ERROR;
     }
 
     ngx_memcpy(frame.u.path_challenge.data, path->challenge2, 8);
 
-    if (ngx_quic_frame_sendto(c, &frame, 1200, path) != NGX_OK) {
+    if (ngx_quic_frame_sendto(c, &frame, 1200, path) == NGX_ERROR) {
         return NGX_ERROR;
     }
 
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -1236,7 +1236,7 @@ ngx_quic_frame_sendto(ngx_connection_t *
 
     sent = ngx_quic_send(c, res.data, res.len, path->sockaddr, path->socklen);
     if (sent < 0) {
-        return NGX_ERROR;
+        return sent;
     }
 
     path->sent += sent;

From arut at nginx.com  Thu Jul  6 13:57:17 2023
From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=)
Date: Thu, 06 Jul 2023 17:57:17 +0400
Subject: [PATCH 4 of 4] QUIC: path MTU discovery
In-Reply-To: <patchbomb.1688651833@arut-laptop>
References: <patchbomb.1688651833@arut-laptop>
Message-ID: <1a49fd5d2013b6c8d502.1688651837@arut-laptop>

# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1688481216 -14400
#      Tue Jul 04 18:33:36 2023 +0400
# Node ID 1a49fd5d2013b6c8d50263499e6ced440927e949
# Parent  a1ea543d009311765144351b2557c00c8e6445bf
QUIC: path MTU discovery.

MTU selection starts by stepping up until the first failure.  Then binary
search is used to find the path MTU.

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -1583,6 +1583,10 @@ ngx_connection_error(ngx_connection_t *c
     }
 #endif
 
+    if (err == NGX_EMSGSIZE && c->log_error == NGX_ERROR_IGNORE_EMSGSIZE) {
+        return 0;
+    }
+
     if (err == 0
         || err == NGX_ECONNRESET
 #if (NGX_WIN32)
@@ -1600,6 +1604,7 @@ ngx_connection_error(ngx_connection_t *c
     {
         switch (c->log_error) {
 
+        case NGX_ERROR_IGNORE_EMSGSIZE:
         case NGX_ERROR_IGNORE_EINVAL:
         case NGX_ERROR_IGNORE_ECONNRESET:
         case NGX_ERROR_INFO:
diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -97,7 +97,8 @@ typedef enum {
     NGX_ERROR_ERR,
     NGX_ERROR_INFO,
     NGX_ERROR_IGNORE_ECONNRESET,
-    NGX_ERROR_IGNORE_EINVAL
+    NGX_ERROR_IGNORE_EINVAL,
+    NGX_ERROR_IGNORE_EMSGSIZE
 } ngx_connection_log_error_e;
 
 
diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
--- a/src/event/quic/ngx_event_quic.c
+++ b/src/event/quic/ngx_event_quic.c
@@ -149,11 +149,6 @@ ngx_quic_apply_transport_params(ngx_conn
         ngx_log_error(NGX_LOG_INFO, c->log, 0,
                       "quic maximum packet size is invalid");
         return NGX_ERROR;
-
-    } else if (ctp->max_udp_payload_size > ngx_quic_max_udp_payload(c)) {
-        ctp->max_udp_payload_size = ngx_quic_max_udp_payload(c);
-        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                       "quic client maximum packet size truncated");
     }
 
     if (ctp->active_connection_id_limit < 2) {
@@ -286,7 +281,7 @@ ngx_quic_new_connection(ngx_connection_t
 
     qc->path_validation.log = c->log;
     qc->path_validation.data = c;
-    qc->path_validation.handler = ngx_quic_path_validation_handler;
+    qc->path_validation.handler = ngx_quic_path_handler;
 
     qc->conf = conf;
 
@@ -297,7 +292,7 @@ ngx_quic_new_connection(ngx_connection_t
     ctp = &qc->ctp;
 
     /* defaults to be used before actual client parameters are received */
-    ctp->max_udp_payload_size = ngx_quic_max_udp_payload(c);
+    ctp->max_udp_payload_size = NGX_QUIC_MAX_UDP_PAYLOAD_SIZE;
     ctp->ack_delay_exponent = NGX_QUIC_DEFAULT_ACK_DELAY_EXPONENT;
     ctp->max_ack_delay = NGX_QUIC_DEFAULT_MAX_ACK_DELAY;
     ctp->active_connection_id_limit = 2;
diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c
--- a/src/event/quic/ngx_event_quic_ack.c
+++ b/src/event/quic/ngx_event_quic_ack.c
@@ -229,6 +229,12 @@ ngx_quic_handle_ack_frame_range(ngx_conn
 
     qc = ngx_quic_get_connection(c);
 
+    if (ctx->level == ssl_encryption_application) {
+        if (ngx_quic_handle_path_mtu_ack(c, qc->path, min, max) != NGX_OK) {
+            return NGX_ERROR;
+        }
+    }
+
     st->max_pn = NGX_TIMER_INFINITE;
     found = 0;
 
diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h
--- a/src/event/quic/ngx_event_quic_connection.h
+++ b/src/event/quic/ngx_event_quic_connection.h
@@ -66,6 +66,14 @@ typedef struct ngx_quic_keys_s        ng
 #define ngx_quic_get_socket(c)               ((ngx_quic_socket_t *)((c)->udp))
 
 
+typedef enum {
+    NGX_QUIC_PATH_IDLE = 0,
+    NGX_QUIC_PATH_VALIDATING,
+    NGX_QUIC_PATH_WAITING,
+    NGX_QUIC_PATH_DISCOVERING_MTU
+} ngx_quic_path_state_e;
+
+
 struct ngx_quic_client_id_s {
     ngx_queue_t                       queue;
     uint64_t                          seqnum;
@@ -89,18 +97,22 @@ struct ngx_quic_path_s {
     ngx_sockaddr_t                    sa;
     socklen_t                         socklen;
     ngx_quic_client_id_t             *cid;
+    ngx_quic_path_state_e             state;
     ngx_msec_t                        expires;
     ngx_uint_t                        tries;
     ngx_uint_t                        tag;
+    size_t                            mtu;
+    size_t                            mtud;
+    size_t                            max_mtu;
     off_t                             sent;
     off_t                             received;
     u_char                            challenge1[8];
     u_char                            challenge2[8];
     uint64_t                          seqnum;
+    uint64_t                          mtu_pnum[NGX_QUIC_PATH_RETRIES];
     ngx_str_t                         addr_text;
     u_char                            text[NGX_SOCKADDR_STRLEN];
-    unsigned                          validated:1;
-    unsigned                          validating:1;
+    ngx_uint_t                        validated; /* unsigned validated:1; */
 };
 
 
diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c
--- a/src/event/quic/ngx_event_quic_migration.c
+++ b/src/event/quic/ngx_event_quic_migration.c
@@ -10,6 +10,11 @@
 #include <ngx_event_quic_connection.h>
 
 
+#define NGX_QUIC_PATH_MTU_DELAY       1000
+#define NGX_QUIC_PATH_MTU_STEP        256
+#define NGX_QUIC_PATH_MTU_PRECISION   16
+
+
 static void ngx_quic_set_connection_path(ngx_connection_t *c,
     ngx_quic_path_t *path);
 static ngx_int_t ngx_quic_validate_path(ngx_connection_t *c,
@@ -17,7 +22,15 @@ static ngx_int_t ngx_quic_validate_path(
 static ngx_int_t ngx_quic_send_path_challenge(ngx_connection_t *c,
     ngx_quic_path_t *path);
 static void ngx_quic_set_path_timer(ngx_connection_t *c);
+static ngx_int_t ngx_quic_expire_path_validation(ngx_connection_t *c,
+    ngx_quic_path_t *path);
+static ngx_int_t ngx_quic_expire_path_mtu_delay(ngx_connection_t *c,
+    ngx_quic_path_t *path);
+static ngx_int_t ngx_quic_expire_path_mtu_discovery(ngx_connection_t *c,
+    ngx_quic_path_t *path);
 static ngx_quic_path_t *ngx_quic_get_path(ngx_connection_t *c, ngx_uint_t tag);
+static ngx_int_t ngx_quic_send_path_mtu_probe(ngx_connection_t *c,
+    ngx_quic_path_t *path);
 
 
 ngx_int_t
@@ -97,7 +110,7 @@ ngx_quic_handle_path_response_frame(ngx_
     {
         path = ngx_queue_data(q, ngx_quic_path_t, queue);
 
-        if (!path->validating) {
+        if (path->state != NGX_QUIC_PATH_VALIDATING) {
             continue;
         }
 
@@ -136,6 +149,9 @@ valid:
         {
             /* address did not change */
             rst = 0;
+
+            path->mtu = prev->mtu;
+            path->max_mtu = prev->max_mtu;
         }
     }
 
@@ -167,9 +183,8 @@ valid:
     ngx_quic_path_dbg(c, "is validated", path);
 
     path->validated = 1;
-    path->validating = 0;
 
-    ngx_quic_set_path_timer(c);
+    ngx_quic_discover_path_mtu(c, path);
 
     return NGX_OK;
 }
@@ -217,6 +232,8 @@ ngx_quic_new_path(ngx_connection_t *c,
     path->addr_text.len = ngx_sock_ntop(sockaddr, socklen, path->text,
                                         NGX_SOCKADDR_STRLEN, 1);
 
+    path->mtu = NGX_QUIC_MIN_INITIAL_SIZE;
+
     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
                    "quic path seq:%uL created addr:%V",
                    path->seqnum, &path->addr_text);
@@ -464,7 +481,7 @@ ngx_quic_handle_migration(ngx_connection
 
     ngx_quic_set_connection_path(c, next);
 
-    if (!next->validated && !next->validating) {
+    if (!next->validated && next->state != NGX_QUIC_PATH_VALIDATING) {
         if (ngx_quic_validate_path(c, next) != NGX_OK) {
             return NGX_ERROR;
         }
@@ -492,7 +509,6 @@ ngx_quic_validate_path(ngx_connection_t 
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
                    "quic initiated validation of path seq:%uL", path->seqnum);
 
-    path->validating = 1;
     path->tries = 0;
 
     if (RAND_bytes(path->challenge1, 8) != 1) {
@@ -511,6 +527,7 @@ ngx_quic_validate_path(ngx_connection_t 
     pto = ngx_max(ngx_quic_pto(c, ctx), 1000);
 
     path->expires = ngx_current_msec + pto;
+    path->state = NGX_QUIC_PATH_VALIDATING;
 
     ngx_quic_set_path_timer(c);
 
@@ -558,6 +575,42 @@ ngx_quic_send_path_challenge(ngx_connect
 }
 
 
+void
+ngx_quic_discover_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path)
+{
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    if (path->max_mtu) {
+        if (path->max_mtu - path->mtu <= NGX_QUIC_PATH_MTU_PRECISION) {
+            path->state = NGX_QUIC_PATH_IDLE;
+            ngx_quic_set_path_timer(c);
+            return;
+        }
+
+        path->mtud = (path->mtu + path->max_mtu) / 2;
+
+    } else {
+        path->mtud = path->mtu + NGX_QUIC_PATH_MTU_STEP;
+
+        if (path->mtud >= qc->ctp.max_udp_payload_size) {
+            path->mtud = qc->ctp.max_udp_payload_size;
+            path->max_mtu = qc->ctp.max_udp_payload_size;
+        }
+    }
+
+    path->state = NGX_QUIC_PATH_WAITING;
+    path->expires = ngx_current_msec + NGX_QUIC_PATH_MTU_DELAY;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic path seq:%uL schedule mtu:%uz",
+                   path->seqnum, path->mtud);
+
+    ngx_quic_set_path_timer(c);
+}
+
+
 static void
 ngx_quic_set_path_timer(ngx_connection_t *c)
 {
@@ -578,7 +631,7 @@ ngx_quic_set_path_timer(ngx_connection_t
     {
         path = ngx_queue_data(q, ngx_quic_path_t, queue);
 
-        if (!path->validating) {
+        if (path->state == NGX_QUIC_PATH_IDLE) {
             continue;
         }
 
@@ -600,22 +653,18 @@ ngx_quic_set_path_timer(ngx_connection_t
 
 
 void
-ngx_quic_path_validation_handler(ngx_event_t *ev)
+ngx_quic_path_handler(ngx_event_t *ev)
 {
     ngx_msec_t              now;
     ngx_queue_t            *q;
-    ngx_msec_int_t          left, next, pto;
-    ngx_quic_path_t        *path, *bkp;
+    ngx_msec_int_t          left;
+    ngx_quic_path_t        *path;
     ngx_connection_t       *c;
-    ngx_quic_send_ctx_t    *ctx;
     ngx_quic_connection_t  *qc;
 
     c = ev->data;
     qc = ngx_quic_get_connection(c);
 
-    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
-
-    next = -1;
     now = ngx_current_msec;
 
     q = ngx_queue_head(&qc->paths);
@@ -625,83 +674,274 @@ ngx_quic_path_validation_handler(ngx_eve
         path = ngx_queue_data(q, ngx_quic_path_t, queue);
         q = ngx_queue_next(q);
 
-        if (!path->validating) {
+        if (path->state == NGX_QUIC_PATH_IDLE) {
             continue;
         }
 
         left = path->expires - now;
 
         if (left > 0) {
-
-            if (next == -1 || left < next) {
-                next = left;
-            }
-
-            continue;
-        }
-
-        if (++path->tries < NGX_QUIC_PATH_RETRIES) {
-            pto = ngx_max(ngx_quic_pto(c, ctx), 1000) << path->tries;
-
-            path->expires = ngx_current_msec + pto;
-
-            if (next == -1 || pto < next) {
-                next = pto;
-            }
-
-            /* retransmit */
-            (void) ngx_quic_send_path_challenge(c, path);
-
             continue;
         }
 
-        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
-                       "quic path seq:%uL validation failed", path->seqnum);
+        switch (path->state) {
+        case NGX_QUIC_PATH_VALIDATING:
+             if (ngx_quic_expire_path_validation(c, path) != NGX_OK) {
+                goto failed;
+            }
+
+            break;
+
+        case NGX_QUIC_PATH_WAITING:
+            if (ngx_quic_expire_path_mtu_delay(c, path) != NGX_OK) {
+                goto failed;
+            }
+
+            break;
+
+        case NGX_QUIC_PATH_DISCOVERING_MTU:
+            if (ngx_quic_expire_path_mtu_discovery(c, path) != NGX_OK) {
+                goto failed;
+             }
+
+            break;
+
+        default:
+            break;
+         }
+     }
+
+    ngx_quic_set_path_timer(c);
+
+    return;
 
-        /* found expired path */
+failed:
+
+    ngx_quic_close_connection(c, NGX_ERROR);
+}
+
+
+static ngx_int_t
+ngx_quic_expire_path_validation(ngx_connection_t *c, ngx_quic_path_t *path)
+{
+    ngx_msec_int_t          pto;
+    ngx_quic_path_t        *bkp;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
 
-        path->validated = 0;
-        path->validating = 0;
+    qc = ngx_quic_get_connection(c);
+    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
+
+    if (++path->tries < NGX_QUIC_PATH_RETRIES) {
+        pto = ngx_max(ngx_quic_pto(c, ctx), 1000) << path->tries;
+        path->expires = ngx_current_msec + pto;
+
+        (void) ngx_quic_send_path_challenge(c, path);
+
+        return NGX_OK;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic path seq:%uL validation failed", path->seqnum);
+
+    /* found expired path */
+
+    path->validated = 0;
 
 
-        /* RFC 9000, 9.3.2.  On-Path Address Spoofing
-         *
-         * To protect the connection from failing due to such a spurious
-         * migration, an endpoint MUST revert to using the last validated
-         * peer address when validation of a new peer address fails.
-         */
-
-        if (qc->path == path) {
-            /* active path validation failed */
-
-            bkp = ngx_quic_get_path(c, NGX_QUIC_PATH_BACKUP);
+    /* RFC 9000, 9.3.2.  On-Path Address Spoofing
+     *
+     * To protect the connection from failing due to such a spurious
+     * migration, an endpoint MUST revert to using the last validated
+     * peer address when validation of a new peer address fails.
+     */
 
-            if (bkp == NULL) {
-                qc->error = NGX_QUIC_ERR_NO_VIABLE_PATH;
-                qc->error_reason = "no viable path";
-                ngx_quic_close_connection(c, NGX_ERROR);
-                return;
-            }
+    if (qc->path == path) {
+        /* active path validation failed */
+
+        bkp = ngx_quic_get_path(c, NGX_QUIC_PATH_BACKUP);
 
-            qc->path = bkp;
-            qc->path->tag = NGX_QUIC_PATH_ACTIVE;
-
-            ngx_quic_set_connection_path(c, qc->path);
-
-            ngx_log_error(NGX_LOG_INFO, c->log, 0,
-                          "quic path seq:%uL addr:%V is restored from backup",
-                          qc->path->seqnum, &qc->path->addr_text);
-
-            ngx_quic_path_dbg(c, "is active", qc->path);
+        if (bkp == NULL) {
+            qc->error = NGX_QUIC_ERR_NO_VIABLE_PATH;
+            qc->error_reason = "no viable path";
+            return NGX_ERROR;
         }
 
-        if (ngx_quic_free_path(c, path) != NGX_OK) {
-            ngx_quic_close_connection(c, NGX_ERROR);
-            return;
+        qc->path = bkp;
+        qc->path->tag = NGX_QUIC_PATH_ACTIVE;
+
+        ngx_quic_set_connection_path(c, qc->path);
+
+        ngx_log_error(NGX_LOG_INFO, c->log, 0,
+                      "quic path seq:%uL addr:%V is restored from backup",
+                      qc->path->seqnum, &qc->path->addr_text);
+
+        ngx_quic_path_dbg(c, "is active", qc->path);
+    }
+
+    return ngx_quic_free_path(c, path);
+}
+
+
+static ngx_int_t
+ngx_quic_expire_path_mtu_delay(ngx_connection_t *c, ngx_quic_path_t *path)
+{
+    ngx_int_t               rc;
+    ngx_uint_t              i;
+    ngx_msec_t              pto;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
+
+    path->tries = 0;
+
+    for ( ;; ) {
+
+        for (i = 0; i < NGX_QUIC_PATH_RETRIES; i++) {
+            path->mtu_pnum[i] = NGX_QUIC_UNSET_PN;
+        }
+
+        rc = ngx_quic_send_path_mtu_probe(c, path);
+
+        if (rc == NGX_ERROR) {
+            return NGX_ERROR;
+        }
+
+        if (rc == NGX_OK) {
+            pto = ngx_quic_pto(c, ctx);
+            path->expires = ngx_current_msec + pto;
+            path->state = NGX_QUIC_PATH_DISCOVERING_MTU;
+            return NGX_OK;
+        }
+
+        /* rc == NGX_DECLINED */
+
+        path->max_mtu = path->mtud;
+
+        if (path->max_mtu - path->mtu <= NGX_QUIC_PATH_MTU_PRECISION) {
+            path->state = NGX_QUIC_PATH_IDLE;
+            return NGX_OK;
+        }
+
+        path->mtud = (path->mtu + path->max_mtu) / 2;
+    }
+}
+
+
+static ngx_int_t
+ngx_quic_expire_path_mtu_discovery(ngx_connection_t *c, ngx_quic_path_t *path)
+{
+    ngx_int_t               rc;
+    ngx_msec_int_t          pto;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
+
+    if (++path->tries < NGX_QUIC_PATH_RETRIES) {
+        pto = ngx_quic_pto(c, ctx) << path->tries;
+        path->expires = ngx_current_msec + pto;
+
+        rc = ngx_quic_send_path_mtu_probe(c, path);
+        if (rc != NGX_DECLINED) {
+            return rc;
         }
     }
 
-    if (next != -1) {
-        ngx_add_timer(&qc->path_validation, next);
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic path seq:%uL expired mtu:%uz",
+                   path->seqnum, path->mtud);
+
+    path->max_mtu = path->mtud;
+
+    ngx_quic_discover_path_mtu(c, path);
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path)
+{
+    ngx_int_t               rc;
+    ngx_uint_t              log_error;
+    ngx_quic_frame_t        frame;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
+
+    ngx_memzero(&frame, sizeof(ngx_quic_frame_t));
+
+    frame.level = ssl_encryption_application;
+    frame.type = NGX_QUIC_FT_PING;
+
+    qc = ngx_quic_get_connection(c);
+    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
+    path->mtu_pnum[path->tries] = ctx->pnum;
+
+    ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic path seq:%uL send probe "
+                   "mtu:%uz pnum:%uL tries:%ui",
+                   path->seqnum, path->mtud, ctx->pnum, path->tries);
+
+    log_error = c->log_error;
+    c->log_error = NGX_ERROR_IGNORE_EMSGSIZE;
+
+    rc = ngx_quic_frame_sendto(c, &frame, path->mtud, path);
+    c->log_error = log_error;
+
+    if (rc == NGX_ERROR) {
+        if (c->write->error) {
+            c->write->error = 0;
+
+            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "quic path seq:%uL rejected mtu:%uz",
+                           path->seqnum, path->mtud);
+
+            return NGX_DECLINED;
+        }
+
+        return NGX_ERROR;
     }
+
+    return NGX_OK;
 }
+
+
+ngx_int_t
+ngx_quic_handle_path_mtu_ack(ngx_connection_t *c, ngx_quic_path_t *path,
+    uint64_t min, uint64_t max)
+{
+    uint64_t    pnum;
+    ngx_uint_t  i;
+
+    if (path->state != NGX_QUIC_PATH_DISCOVERING_MTU) {
+        return NGX_OK;
+    }
+
+    for (i = 0; i < NGX_QUIC_PATH_RETRIES; i++) {
+        pnum = path->mtu_pnum[i];
+
+        if (pnum == NGX_QUIC_UNSET_PN) {
+            break;
+        }
+
+        if (pnum < min || pnum > max) {
+            continue;
+        }
+
+        path->mtu = path->mtud;
+
+        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "quic path seq:%uL ack mtu:%uz",
+                       path->seqnum, path->mtu);
+
+        ngx_quic_discover_path_mtu(c, path);
+
+        break;
+    }
+
+    return NGX_OK;
+}
diff --git a/src/event/quic/ngx_event_quic_migration.h b/src/event/quic/ngx_event_quic_migration.h
--- a/src/event/quic/ngx_event_quic_migration.h
+++ b/src/event/quic/ngx_event_quic_migration.h
@@ -18,10 +18,10 @@
 #define NGX_QUIC_PATH_BACKUP    2
 
 #define ngx_quic_path_dbg(c, msg, path)                                       \
-    ngx_log_debug6(NGX_LOG_DEBUG_EVENT, c->log, 0,                            \
-                   "quic path seq:%uL %s sent:%O recvd:%O state:%s%s",        \
+    ngx_log_debug7(NGX_LOG_DEBUG_EVENT, c->log, 0,                            \
+                   "quic path seq:%uL %s tx:%O rx:%O valid:%d st:%d mtu:%uz", \
                    path->seqnum, msg, path->sent, path->received,             \
-                   path->validated ? "V": "N", path->validating ? "R": "");
+                   path->validated, path->state, path->mtu);
 
 ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c,
     ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f);
@@ -36,6 +36,10 @@ ngx_int_t ngx_quic_set_path(ngx_connecti
 ngx_int_t ngx_quic_handle_migration(ngx_connection_t *c,
     ngx_quic_header_t *pkt);
 
-void ngx_quic_path_validation_handler(ngx_event_t *ev);
+void ngx_quic_path_handler(ngx_event_t *ev);
+
+void ngx_quic_discover_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path);
+ngx_int_t ngx_quic_handle_path_mtu_ack(ngx_connection_t *c,
+    ngx_quic_path_t *path, uint64_t min, uint64_t max);
 
 #endif /* _NGX_EVENT_QUIC_MIGRATION_H_INCLUDED_ */
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -10,9 +10,6 @@
 #include <ngx_event_quic_connection.h>
 
 
-#define NGX_QUIC_MAX_UDP_PAYLOAD_OUT   1252
-#define NGX_QUIC_MAX_UDP_PAYLOAD_OUT6  1232
-
 #define NGX_QUIC_MAX_UDP_SEGMENT_BUF  65487 /* 65K - IPv6 header */
 #define NGX_QUIC_MAX_SEGMENTS            64 /* UDP_MAX_SEGMENTS */
 
@@ -61,21 +58,6 @@ static size_t ngx_quic_path_limit(ngx_co
     size_t size);
 
 
-size_t
-ngx_quic_max_udp_payload(ngx_connection_t *c)
-{
-    /* TODO: path MTU discovery */
-
-#if (NGX_HAVE_INET6)
-    if (c->sockaddr->sa_family == AF_INET6) {
-        return NGX_QUIC_MAX_UDP_PAYLOAD_OUT6;
-    }
-#endif
-
-    return NGX_QUIC_MAX_UDP_PAYLOAD_OUT;
-}
-
-
 ngx_int_t
 ngx_quic_output(ngx_connection_t *c)
 {
@@ -142,10 +124,7 @@ ngx_quic_create_datagrams(ngx_connection
 
         p = dst;
 
-        len = ngx_min(qc->ctp.max_udp_payload_size,
-                      NGX_QUIC_MAX_UDP_PAYLOAD_SIZE);
-
-        len = ngx_quic_path_limit(c, path, len);
+        len = ngx_quic_path_limit(c, path, path->mtu);
 
         pad = ngx_quic_get_padding_level(c);
 
@@ -299,9 +278,7 @@ ngx_quic_allow_segmentation(ngx_connecti
     ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
 
     bytes = 0;
-
-    len = ngx_min(qc->ctp.max_udp_payload_size,
-                  NGX_QUIC_MAX_UDP_SEGMENT_BUF);
+    len = ngx_min(qc->path->mtu, NGX_QUIC_MAX_UDP_SEGMENT_BUF);
 
     for (q = ngx_queue_head(&ctx->frames);
          q != ngx_queue_sentinel(&ctx->frames);
@@ -345,8 +322,7 @@ ngx_quic_create_segments(ngx_connection_
         return NGX_ERROR;
     }
 
-    segsize = ngx_min(qc->ctp.max_udp_payload_size,
-                      NGX_QUIC_MAX_UDP_SEGMENT_BUF);
+    segsize = ngx_min(path->mtu, NGX_QUIC_MAX_UDP_SEGMENT_BUF);
     p = dst;
     end = dst + sizeof(dst);
 
diff --git a/src/event/quic/ngx_event_quic_output.h b/src/event/quic/ngx_event_quic_output.h
--- a/src/event/quic/ngx_event_quic_output.h
+++ b/src/event/quic/ngx_event_quic_output.h
@@ -12,8 +12,6 @@
 #include <ngx_core.h>
 
 
-size_t ngx_quic_max_udp_payload(ngx_connection_t *c);
-
 ngx_int_t ngx_quic_output(ngx_connection_t *c);
 
 ngx_int_t ngx_quic_negotiate_version(ngx_connection_t *c,
diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c
--- a/src/event/quic/ngx_event_quic_ssl.c
+++ b/src/event/quic/ngx_event_quic_ssl.c
@@ -494,6 +494,8 @@ ngx_quic_crypto_input(ngx_connection_t *
      */
     ngx_quic_discard_ctx(c, ssl_encryption_handshake);
 
+    ngx_quic_discover_path_mtu(c, qc->path);
+
     /* start accepting clients on negotiated number of server ids */
     if (ngx_quic_create_sockets(c) != NGX_OK) {
         return NGX_ERROR;
diff --git a/src/os/unix/ngx_errno.h b/src/os/unix/ngx_errno.h
--- a/src/os/unix/ngx_errno.h
+++ b/src/os/unix/ngx_errno.h
@@ -54,6 +54,7 @@ typedef int               ngx_err_t;
 #define NGX_ENOMOREFILES  0
 #define NGX_ELOOP         ELOOP
 #define NGX_EBADF         EBADF
+#define NGX_EMSGSIZE      EMSGSIZE
 
 #if (NGX_HAVE_OPENAT)
 #define NGX_EMLINK        EMLINK

From xeioex at nginx.com  Thu Jul  6 16:30:23 2023
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 06 Jul 2023 16:30:23 +0000
Subject: [njs] Version 0.8.0.
Message-ID: <hg.0ed1952588ab.1688661023.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/0ed1952588ab
branches:  
changeset: 2177:0ed1952588ab
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Wed Jul 05 17:49:50 2023 -0700
description:
Version 0.8.0.

diffstat:

 CHANGES |  76 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 76 insertions(+), 0 deletions(-)

diffs (83 lines):

diff -r ff7eb3c4bf76 -r 0ed1952588ab CHANGES
--- a/CHANGES	Mon Jul 03 13:32:41 2023 -0700
+++ b/CHANGES	Wed Jul 05 17:49:50 2023 -0700
@@ -1,3 +1,79 @@
+Changes with njs 0.8.0                                        6 Jul 2023
+
+    nginx modules:
+
+    *) Change: removed special treatment of forbidden headers in Fetch API
+       introduced in 0.7.10.
+
+    *) Change: removed deprecated since 0.5.0 r.requestBody and
+       r.responseBody in HTTP module.
+
+    *) Change: throwing an exception in r.internalRedirect() while
+       filtering in HTTP module.
+
+    *) Feature: introduced global nginx properties.
+        ngx.build - an optional nginx build name, corresponds to
+        --build=name argument of configure script, by default is "".
+        ngx.conf_file_path - the file path to current nginx configuration
+            file.
+        ngx.error_log_path - the file path to current error log file.
+        ngx.prefix - the directory that keeps server files.
+        ngx.version - the nginx version as a string, for example: "1.25.0".
+        ngx.version_number - the nginx version as a number, for example:
+            1025000.
+        ngx.worker_id - corresponds to an nginx internal worker id.
+           The value is between 0 and worker_processes - 1.
+
+    *) Feature: introduced js_shared_dict_zone directive.
+        The directive allows to declare a dictionary that is shared among the
+        working processes.
+
+    *) Improvement: added compile-time options to disable njs modules.
+        For example to disable libxslt related code:
+        NJS_LIBXSLT=NO ./configure  .. --add-module=/path/to/njs/module
+
+    *) Bugfix: fixed r.status setter when filtering in HTTP module.
+
+    *) Bugfix: fixed setting of Location header in HTTP module.
+
+    Core:
+
+    *) Change: native methods are provided with retval argument.
+       This change breaks compatibility with C extension for njs
+       requiring to modify the code.
+
+    *) Change: non-compliant deprecated String methods were removed.
+        The following methods were removed: String.bytesFrom(),
+        String.prototype.fromBytes(), String.prototype.fromUTF8(),
+        String.prototype.toBytes(), String.prototype.toUTF8(),
+        String.prototype.toString(encoding).
+
+    *) Change: removed support for building with GNU readline.
+
+    *) Feature: added Array.from(), Array.prototype.toSorted(),
+        Array.prototype.toSpliced(), Array.prototype.toReversed().
+
+    *) Feature: added %TypedArray%.prototype.toSorted(),
+        %TypedArray%.prototype.toSpliced(),
+        %TypedArray%.prototype.toReversed().
+
+    *) Feature: added CryptoKey properties in WebCrypto.
+        The following properties for CryptoKey were added:
+        algorithm, extractable, type, usages.
+
+    *) Bugfix: fixed retval of crypto.getRandomValues().
+
+    *) Bugfix: fixed evaluation of computed property names with function
+       expressions.
+
+    *) Bugfix: fixed implicit name for a function expression declared in
+       arrays.
+
+    *) Bugfix: fixed parsing of for-in loops.
+
+    *) Bugfix: fixed Date.parse() with ISO-8601 format and UTC time
+       offset.
+
 Changes with njs 0.7.12                                       10 Apr 2023
 
     nginx modules:

From xeioex at nginx.com  Thu Jul  6 16:30:25 2023
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 06 Jul 2023 16:30:25 +0000
Subject: [njs] Added tag 0.8.0 for changeset 0ed1952588ab
Message-ID: <hg.034ad86967b6.1688661025.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/034ad86967b6
branches:  
changeset: 2178:034ad86967b6
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu Jul 06 09:10:47 2023 -0700
description:
Added tag 0.8.0 for changeset 0ed1952588ab

diffstat:

 .hgtags |  1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diffs (8 lines):

diff -r 0ed1952588ab -r 034ad86967b6 .hgtags
--- a/.hgtags	Wed Jul 05 17:49:50 2023 -0700
+++ b/.hgtags	Thu Jul 06 09:10:47 2023 -0700
@@ -62,3 +62,4 @@ 0000000000000000000000000000000000000000
 3a1b46d51f040f5e7b9b81c3b2b312a2d272f0a3 0.7.10
 26dd3824b9f343e2768609c1b673f788e3a5e154 0.7.11
 a1faa64d4972020413fd168e2b542bcc150819c0 0.7.12
+0ed1952588ab1e0e1c18425fe7923b2b76f38a65 0.8.0

From arut at nginx.com  Fri Jul  7 07:58:13 2023
From: arut at nginx.com (Roman Arutyunyan)
Date: Fri, 7 Jul 2023 11:58:13 +0400
Subject: [PATCH 4 of 4] QUIC: path MTU discovery
In-Reply-To: <1a49fd5d2013b6c8d502.1688651837@arut-laptop>
References: <patchbomb.1688651833@arut-laptop>
 <1a49fd5d2013b6c8d502.1688651837@arut-laptop>
Message-ID: <20230707075813.eigmowojyp7gya2d@N00W24XTQX>

On Thu, Jul 06, 2023 at 05:57:17PM +0400, Roman Arutyunyan wrote:
> # HG changeset patch
> # User Roman Arutyunyan <arut at nginx.com>
> # Date 1688481216 -14400
> #      Tue Jul 04 18:33:36 2023 +0400
> # Node ID 1a49fd5d2013b6c8d50263499e6ced440927e949
> # Parent  a1ea543d009311765144351b2557c00c8e6445bf
> QUIC: path MTU discovery.
> 
> MTU selection starts by stepping up until the first failure.  Then binary
>in search is used to find the path MTU.

[..]

As an alternative, the ascending part can be changed from additive to
multiplicative, which would work better on the off chance MTU is really high.

diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c
--- a/src/event/quic/ngx_event_quic_migration.c
+++ b/src/event/quic/ngx_event_quic_migration.c
@@ -11,7 +11,6 @@
 
 
 #define NGX_QUIC_PATH_MTU_DELAY       1000
-#define NGX_QUIC_PATH_MTU_STEP        256
 #define NGX_QUIC_PATH_MTU_PRECISION   16
 
 
@@ -592,7 +591,7 @@ ngx_quic_discover_path_mtu(ngx_connectio
         path->mtud = (path->mtu + path->max_mtu) / 2;
 
     } else {
-        path->mtud = path->mtu + NGX_QUIC_PATH_MTU_STEP;
+        path->mtud = path->mtu * 2;
 
         if (path->mtud >= qc->ctp.max_udp_payload_size) {
             path->mtud = qc->ctp.max_udp_payload_size;

--
Roman Arutyunyan

From eero.aaltonen at vaisala.com  Fri Jul  7 15:02:14 2023
From: eero.aaltonen at vaisala.com (eero.aaltonen at vaisala.com)
Date: Fri,  7 Jul 2023 18:02:14 +0300
Subject: [RFC][PATCH 0/1] Add option to use directory for trusted CAs
Message-ID: <20230707150215.406702-1-eero.aaltonen@vaisala.com>

From: Eero Aaltonen <eero.aaltonen at vaisala.com>

I was looking for an option to configure the trusted CAs using a directory,
equivalent to the OpenSSL -CApath option. The option seemed to be missing, so
here's a minimal working example of what I would like to accomplish.

The current version is still missing code to populate the list used for
SSL_CTX_set_client_CA_list, but enough to actually verify a certificate chain
using CAs in the 'ssl_client_ca_dir' specified directory.

Comments appreciated.

-- 
Eero

Eero Aaltonen (1):
  WIP: SSL: add ssl_client_ca_dir option for trusted CAs

 src/event/ngx_event_openssl.c            | 24 +++++++++++++++++-------
 src/event/ngx_event_openssl.h            |  2 +-
 src/http/modules/ngx_http_grpc_module.c  |  1 +
 src/http/modules/ngx_http_proxy_module.c |  1 +
 src/http/modules/ngx_http_ssl_module.c   | 15 +++++++++++++--
 src/http/modules/ngx_http_ssl_module.h   |  1 +
 src/http/modules/ngx_http_uwsgi_module.c |  1 +
 src/mail/ngx_mail_ssl_module.c           |  5 +++--
 src/stream/ngx_stream_proxy_module.c     |  1 +
 src/stream/ngx_stream_ssl_module.c       |  5 +++--
 src/stream/ngx_stream_ssl_module.h       |  1 +
 11 files changed, 43 insertions(+), 14 deletions(-)

-- 
2.25.1

From eero.aaltonen at vaisala.com  Fri Jul  7 15:02:15 2023
From: eero.aaltonen at vaisala.com (eero.aaltonen at vaisala.com)
Date: Fri,  7 Jul 2023 18:02:15 +0300
Subject: [RFC][PATCH 1/1] Add option to use directory for trusted CAs
In-Reply-To: <20230707150215.406702-1-eero.aaltonen@vaisala.com>
References: <20230707150215.406702-1-eero.aaltonen@vaisala.com>
Message-ID: <20230707150215.406702-2-eero.aaltonen@vaisala.com>

From: Eero Aaltonen <eero.aaltonen at vaisala.com>

Allow configuring trusted CAs to a directory with certificates and hash
symlinks generated with `openssl rehash`.

Signed-off-by: Eero Aaltonen <eero.aaltonen at vaisala.com>
---
 src/event/ngx_event_openssl.c            | 24 +++++++++++++++++-------
 src/event/ngx_event_openssl.h            |  2 +-
 src/http/modules/ngx_http_grpc_module.c  |  1 +
 src/http/modules/ngx_http_proxy_module.c |  1 +
 src/http/modules/ngx_http_ssl_module.c   | 15 +++++++++++++--
 src/http/modules/ngx_http_ssl_module.h   |  1 +
 src/http/modules/ngx_http_uwsgi_module.c |  1 +
 src/mail/ngx_mail_ssl_module.c           |  5 +++--
 src/stream/ngx_stream_proxy_module.c     |  1 +
 src/stream/ngx_stream_ssl_module.c       |  5 +++--
 src/stream/ngx_stream_ssl_module.h       |  1 +
 11 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index d762d6b..3da406c 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -870,24 +870,31 @@ ngx_ssl_ciphers(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *ciphers,
 
 ngx_int_t
 ngx_ssl_client_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
-    ngx_int_t depth)
+    ngx_str_t *ca_dir, ngx_int_t depth)
 {
     STACK_OF(X509_NAME)  *list;
+    char *ca_file = NULL;
+    char *ca_path = NULL;
 
     SSL_CTX_set_verify(ssl->ctx, SSL_VERIFY_PEER, ngx_ssl_verify_callback);
 
     SSL_CTX_set_verify_depth(ssl->ctx, depth);
 
-    if (cert->len == 0) {
+    if (cert->len == 0 && ca_dir->len == 0) {
         return NGX_OK;
     }
 
-    if (ngx_conf_full_name(cf->cycle, cert, 1) != NGX_OK) {
-        return NGX_ERROR;
+    if (ca_dir->len != 0) {
+        ca_path = ca_dir->data;
+    }
+    if (cert->len != 0) {
+        ca_file = cert->data;
+        if (ngx_conf_full_name(cf->cycle, cert, 1) != NGX_OK) {
+            return NGX_ERROR;
+        }
     }
 
-    if (SSL_CTX_load_verify_locations(ssl->ctx, (char *) cert->data, NULL)
-        == 0)
+    if (SSL_CTX_load_verify_locations(ssl->ctx, ca_file, ca_path) == 0)
     {
         ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
                       "SSL_CTX_load_verify_locations(\"%s\") failed",
@@ -902,6 +909,10 @@ ngx_ssl_client_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
 
     ERR_clear_error();
 
+    if (cert->len == 0) { // TODO: load CAs from ca_dir?
+        return NGX_OK;
+    }
+
     list = SSL_load_client_CA_file((char *) cert->data);
 
     if (list == NULL) {
@@ -915,7 +926,6 @@ ngx_ssl_client_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
     return NGX_OK;
 }
 
-
 ngx_int_t
 ngx_ssl_trusted_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
     ngx_int_t depth)
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
index 329760d..0a3481e 100644
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -179,7 +179,7 @@ ngx_int_t ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool,
 ngx_int_t ngx_ssl_ciphers(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *ciphers,
     ngx_uint_t prefer_server_ciphers);
 ngx_int_t ngx_ssl_client_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl,
-    ngx_str_t *cert, ngx_int_t depth);
+    ngx_str_t *cert, ngx_str_t *ca_dir, ngx_int_t depth);
 ngx_int_t ngx_ssl_trusted_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl,
     ngx_str_t *cert, ngx_int_t depth);
 ngx_int_t ngx_ssl_crl(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *crl);
diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c
index 53bc547..d5728e6 100644
--- a/src/http/modules/ngx_http_grpc_module.c
+++ b/src/http/modules/ngx_http_grpc_module.c
@@ -35,6 +35,7 @@ typedef struct {
     ngx_uint_t                 ssl_protocols;
     ngx_str_t                  ssl_ciphers;
     ngx_uint_t                 ssl_verify_depth;
+    ngx_str_t                  ssl_ca_dir;
     ngx_str_t                  ssl_trusted_certificate;
     ngx_str_t                  ssl_crl;
     ngx_str_t                  ssl_certificate;
diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
index a63c3ed..6dda907 100644
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -122,6 +122,7 @@ typedef struct {
     ngx_uint_t                     ssl_protocols;
     ngx_str_t                      ssl_ciphers;
     ngx_uint_t                     ssl_verify_depth;
+    ngx_str_t                      ssl_ca_dir;
     ngx_str_t                      ssl_trusted_certificate;
     ngx_str_t                      ssl_crl;
     ngx_str_t                      ssl_certificate;
diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c
index a47d696..c80b614 100644
--- a/src/http/modules/ngx_http_ssl_module.c
+++ b/src/http/modules/ngx_http_ssl_module.c
@@ -182,6 +182,13 @@ static ngx_command_t  ngx_http_ssl_commands[] = {
       offsetof(ngx_http_ssl_srv_conf_t, client_certificate),
       NULL },
 
+    { ngx_string("ssl_client_ca_dir"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_str_slot,
+      NGX_HTTP_SRV_CONF_OFFSET,
+      offsetof(ngx_http_ssl_srv_conf_t, client_ca_dir),
+      NULL },
+
     { ngx_string("ssl_trusted_certificate"),
       NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_str_slot,
@@ -609,6 +616,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf)
      *     sscf->dhparam = { 0, NULL };
      *     sscf->ecdh_curve = { 0, NULL };
      *     sscf->client_certificate = { 0, NULL };
+     *     sscf->client_ca_dir = { 0, NULL };
      *     sscf->trusted_certificate = { 0, NULL };
      *     sscf->crl = { 0, NULL };
      *     sscf->ciphers = { 0, NULL };
@@ -690,6 +698,8 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
 
     ngx_conf_merge_str_value(conf->client_certificate, prev->client_certificate,
                          "");
+    ngx_conf_merge_str_value(conf->client_ca_dir,
+                         prev->client_ca_dir, "");
     ngx_conf_merge_str_value(conf->trusted_certificate,
                          prev->trusted_certificate, "");
     ngx_conf_merge_str_value(conf->crl, prev->crl, "");
@@ -840,14 +850,15 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
 
     if (conf->verify) {
 
-        if (conf->client_certificate.len == 0 && conf->verify != 3) {
+        if (conf->verify != 3 && conf->client_certificate.len == 0 && conf->client_ca_dir.len == 0) {
             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
-                          "no ssl_client_certificate for ssl_verify_client");
+                          "ssl_verify_client needs ssl_client_certificate or ssl_client_ca_dir");
             return NGX_CONF_ERROR;
         }
 
         if (ngx_ssl_client_certificate(cf, &conf->ssl,
                                        &conf->client_certificate,
+                                       &conf->client_ca_dir,
                                        conf->verify_depth)
             != NGX_OK)
         {
diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h
index 7ab0f7e..e90b79b 100644
--- a/src/http/modules/ngx_http_ssl_module.h
+++ b/src/http/modules/ngx_http_ssl_module.h
@@ -43,6 +43,7 @@ typedef struct {
     ngx_str_t                       dhparam;
     ngx_str_t                       ecdh_curve;
     ngx_str_t                       client_certificate;
+    ngx_str_t                       client_ca_dir;
     ngx_str_t                       trusted_certificate;
     ngx_str_t                       crl;
 
diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c
index 1334f44..7e13a08 100644
--- a/src/http/modules/ngx_http_uwsgi_module.c
+++ b/src/http/modules/ngx_http_uwsgi_module.c
@@ -52,6 +52,7 @@ typedef struct {
     ngx_uint_t                 ssl_protocols;
     ngx_str_t                  ssl_ciphers;
     ngx_uint_t                 ssl_verify_depth;
+    ngx_str_t                  ssl_ca_dir;
     ngx_str_t                  ssl_trusted_certificate;
     ngx_str_t                  ssl_crl;
     ngx_str_t                  ssl_certificate;
diff --git a/src/mail/ngx_mail_ssl_module.c b/src/mail/ngx_mail_ssl_module.c
index 7eae83e..062d4d7 100644
--- a/src/mail/ngx_mail_ssl_module.c
+++ b/src/mail/ngx_mail_ssl_module.c
@@ -403,14 +403,15 @@ ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child)
 
     if (conf->verify) {
 
-        if (conf->client_certificate.len == 0 && conf->verify != 3) {
+        if (conf->verify != 3 && conf->client_certificate.len == 0 && conf->client_ca_dir.len == 0) {
             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
-                          "no ssl_client_certificate for ssl_verify_client");
+                          "ssl_verify_client needs ssl_client_certificate or ssl_client_ca_dir");
             return NGX_CONF_ERROR;
         }
 
         if (ngx_ssl_client_certificate(cf, &conf->ssl,
                                        &conf->client_certificate,
+                                       &conf->client_ca_dir,
                                        conf->verify_depth)
             != NGX_OK)
         {
diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c
index 01cda7a..1ad4ceb 100644
--- a/src/stream/ngx_stream_proxy_module.c
+++ b/src/stream/ngx_stream_proxy_module.c
@@ -44,6 +44,7 @@ typedef struct {
 
     ngx_flag_t                       ssl_verify;
     ngx_uint_t                       ssl_verify_depth;
+    ngx_str_t                        ssl_ca_dir;
     ngx_str_t                        ssl_trusted_certificate;
     ngx_str_t                        ssl_crl;
     ngx_str_t                        ssl_certificate;
diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c
index d8c0471..adaa825 100644
--- a/src/stream/ngx_stream_ssl_module.c
+++ b/src/stream/ngx_stream_ssl_module.c
@@ -761,14 +761,15 @@ ngx_stream_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child)
 
     if (conf->verify) {
 
-        if (conf->client_certificate.len == 0 && conf->verify != 3) {
+        if (conf->verify != 3 && conf->client_certificate.len == 0 && conf->client_ca_dir.len == 0) {
             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
-                          "no ssl_client_certificate for ssl_verify_client");
+                          "ssl_verify_client needs ssl_client_certificate or ssl_client_ca_dir");
             return NGX_CONF_ERROR;
         }
 
         if (ngx_ssl_client_certificate(cf, &conf->ssl,
                                        &conf->client_certificate,
+                                       &conf->client_ca_dir,
                                        conf->verify_depth)
             != NGX_OK)
         {
diff --git a/src/stream/ngx_stream_ssl_module.h b/src/stream/ngx_stream_ssl_module.h
index c6e24be..607146e 100644
--- a/src/stream/ngx_stream_ssl_module.h
+++ b/src/stream/ngx_stream_ssl_module.h
@@ -40,6 +40,7 @@ typedef struct {
     ngx_str_t        dhparam;
     ngx_str_t        ecdh_curve;
     ngx_str_t        client_certificate;
+    ngx_str_t        client_ca_dir;
     ngx_str_t        trusted_certificate;
     ngx_str_t        crl;
 
-- 
2.25.1

From mdounin at mdounin.ru  Fri Jul  7 16:59:00 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Fri, 7 Jul 2023 19:59:00 +0300
Subject: [RFC][PATCH 0/1] Add option to use directory for trusted CAs
In-Reply-To: <20230707150215.406702-1-eero.aaltonen@vaisala.com>
References: <20230707150215.406702-1-eero.aaltonen@vaisala.com>
Message-ID: <ZKhEVMDym9oL2cuZ@mdounin.ru>

Hello!

On Fri, Jul 07, 2023 at 06:02:14PM +0300, Eero Aaltonen via nginx-devel wrote:

> From: Eero Aaltonen <eero.aaltonen at vaisala.com>
> 
> I was looking for an option to configure the trusted CAs using a directory,
> equivalent to the OpenSSL -CApath option. The option seemed to be missing, so
> here's a minimal working example of what I would like to accomplish.
> 
> The current version is still missing code to populate the list used for
> SSL_CTX_set_client_CA_list, but enough to actually verify a certificate chain
> using CAs in the 'ssl_client_ca_dir' specified directory.
> 
> Comments appreciated.

The option to configure CAs using a directory is missing 
intentionally, as loading relevant CA certificates into memory is 
expected to be more efficient than checking things on disk on each 
connection.

If you are nevertheless interested in configuring a directory, 
consider using ssl_conf_command with VerifyCAPath/ClientCAPath 
(https://nginx.org/r/ssl_conf_command, 
https://nginx.org/r/proxy_ssl_conf_command)).

-- 
Maxim Dounin
http://mdounin.ru/

From mdounin at mdounin.ru  Fri Jul  7 23:21:51 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Sat, 8 Jul 2023 02:21:51 +0300
Subject: Fixing the well Documented Nginx Alias Traversal Vulnerability?
In-Reply-To: <CAJq_=OXAOdULC4jnngiLJefcJNR6_hFrL0+34wAuCr-7W7hLkg@mail.gmail.com>
References: <CAJq_=OXAOdULC4jnngiLJefcJNR6_hFrL0+34wAuCr-7W7hLkg@mail.gmail.com>
Message-ID: <ZKieDyPi5c8PXdiP@mdounin.ru>

Hello!

On Thu, Jul 06, 2023 at 12:05:25PM -0400, Jonathan Leitschuh wrote:

> Hi Nginx Team,
> 
> My name is Jonathan Leitschuh. I'm a Senior Software Security Researcher
> for the Open Source Security Foundation: Project Alpha-Omega
> <https://openssf.org/community/alpha-omega/>. “Alpha” works with the
> maintainers of the most critical open source projects to help them identify
> and fix security vulnerabilities, and improve their security posture.
> “Omega” identified at least 10,000 widely deployed OSS projects where it
> can apply automated security analysis, scoring, and remediation guidance to
> their open source maintainer communities.
> 
> Earlier today, on Twitter, I saw this discussion regarding
> the long-standing the NGinx Alias traversal vulnerability:
> https://twitter.com/infosec_au/status/1676072447156834304?s=20
> 
> In particular, this discussion centered on this, this recent blog post
> published on the 3rd of this month.
> https://labs.hakaioffsec.com/nginx-alias-traversal/
> 
> In that blog post, the researchers detail how, due to misconfiguration and
> failing to specify the `/` character in the correct place can lead to path
> traversal vulnerabilities.
> 
> 
> 
> This vulnerability was previously discussed by Detectify in a blog post in
> 2020:
> https://blog.detectify.com/2020/11/10/common-nginx-misconfigurations/
> 
> This research originated with Orange Tsai back in 2018:
> https://i.blackhat.com/us-18/Wed-August-8/us-18-Orange-Tsai-Breaking-Parser-Logic-Take-Your-Path-Normalization-Off-And-Pop-0days-Out-2.pdf
> 
> The recent hakaioffsec blog also identifies this case where the
> vulnerability also appears:
> 
> 
> My question to both the Nginx Security team, and the Nginx Developers, why
> not fix this vulnerability in Nginx itself such that it's safe by default,
> but users can opt-in to the vulnerable behaviour if they so desire through
> an explicit flag? Surely it would be fairly straightforward to identify
> these vulnerable configurations and fix them for the user, or fail closed
> (safe)?
> 
> If you need evidence of the impact this vulnerability can have, the
> above hakaioffsec blog post details how this vulnerability resulted in
> a US$6000 bounty from BitWarden.
> 
> This GitHub search, proposed by the above blog post, returns 2.2k results
> on GitHub:
> 
> https://github.com/search?q=%2Flocation+%5C%2F%5B_.a-zA-Z0-9-%5C%2F%5D*%5B%5E%5C%2F%5D%5B%5Cs%5D%5C%7B%5B%5Cs%5Cn%5D*alias+%5C%2F%5B_.a-zA-Z0-9-%5C%2F%5D*%5C%2F%3B%2F+&type=code
> 
> I'm curious what is currently preventing this from being fixed upstream to
> prevent end-users from accidentally implementing vulnerable configurations
> like this.

Thank you for your message.

Indeed, the misconfiguration in question is a well-known one.  
Gixy, a tool to detect various nginx misconfigurations, tries to 
detect it at least since 2017:

https://github.com/yandex/gixy/blob/master/docs/en/plugins/aliastraversal.md

The root cause as I see it is that nginx locations (and alias) 
work with prefix strings, while unaware people often expect them 
to work with path components (directories) instead.  Note the 
"Achieving Impact Without a Slash on Alias" section in the article 
you've referenced: a configuration like

    location /img {
        alias /var/images;
    }

makes it possible to access "/var/images_confidential" via a 
request to "/img_confidential".  This is because locations works 
on prefix strings, and due to lack of trailing "/" in the location 
prefix it does match "/img_confidential" request, and maps it to 
the file system according to the configuration.

Similarly, the same misconfiguration can happen even without the 
"alias" directive at all. Consider the configuration:

    location / { deny all; }
    location /images { allow all; }

In contrast to some might (incorrectly) assume, a request to 
"/images_confidential" will be allowed, and will return content as 
per the configuration.

Unfortunately, I don't think there is an easy fix.  Changing the 
way how nginx locations work would be a major change, and will 
break a lot of valid configurations where prefix matching is 
intentionally used.

Meanwhile, it surely deserves some additional warnings/notes in 
the documentation, to emphasize the actual behaviour and potential 
pitfalls.  Hopefully it would be enough to at least reduce the 
amount of such misconfigurations.

If you think there is an easy fix, feel free to provide 
suggestions.

(It looks like you aren't subscribed to nginx-devel@, so the 
message did not reach the mailing list: it only accepts messages 
from subscribers.  I've preserved it in Cc: though, so we can 
continue discussion there, with more developers involved.)

-- 
Maxim Dounin
http://mdounin.ru/

From andrew.zv at gmail.com  Sun Jul  9 08:57:20 2023
From: andrew.zv at gmail.com (andrey zverev)
Date: Sun, 9 Jul 2023 10:57:20 +0200
Subject: rbtree redundant assignment
Message-ID: <CAMw0no8J26L=V5MA7ZH+Q7O0qS2EuuXJ-ioThuOKu51do0nczg@mail.gmail.com>

Hello, everyone!

I found a very strange thing in the ngx_rbtree_delete function. It seems
that there is redundant assignment under the following if statement:

if (subst->parent == node) {
  temp->parent = subst;

Could someone please explain to me why this assignment is needed if temp
was taken as subst->right before? Because it was a child of subst, it means
temp had subst as its parent.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20230709/75a33466/attachment.htm>

From mdounin at mdounin.ru  Sun Jul  9 12:24:57 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Sun, 9 Jul 2023 15:24:57 +0300
Subject: rbtree redundant assignment
In-Reply-To: <CAMw0no8J26L=V5MA7ZH+Q7O0qS2EuuXJ-ioThuOKu51do0nczg@mail.gmail.com>
References: <CAMw0no8J26L=V5MA7ZH+Q7O0qS2EuuXJ-ioThuOKu51do0nczg@mail.gmail.com>
Message-ID: <ZKqnGZYpisA15E0h@mdounin.ru>

Hello!

On Sun, Jul 09, 2023 at 10:57:20AM +0200, andrey zverev wrote:

> Hello, everyone!
> 
> I found a very strange thing in the ngx_rbtree_delete function. It seems
> that there is redundant assignment under the following if statement:
> 
> if (subst->parent == node) {
>   temp->parent = subst;
> 
> Could someone please explain to me why this assignment is needed if temp
> was taken as subst->right before? Because it was a child of subst, it means
> temp had subst as its parent.

If subst->right (and therefore temp) is the sentinel, this won't 
be the case, and an explicit assignment is needed.

Just in case, "Introduction to Algorithms" by Cormen et al. might 
be a good source to refer here.

-- 
Maxim Dounin
http://mdounin.ru/

From mdounin at mdounin.ru  Sun Jul  9 21:24:25 2023
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Mon, 10 Jul 2023 00:24:25 +0300
Subject: [PATCH] Tests: enabled TLSv1 in ssl_sni_reneg.t
Message-ID: <85188791cd9cf688a294.1688937865@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1688521184 -10800
#      Wed Jul 05 04:39:44 2023 +0300
# Node ID 85188791cd9cf688a29401e31221551345b76ff4
# Parent  c5767845481fc1d7df3e56b604fc4afdeab7be85
Tests: enabled TLSv1 in ssl_sni_reneg.t.

This fixes running the test with OpenSSL before 1.0.1, where TLSv1.2
support was introduced.

diff --git a/ssl_sni_reneg.t b/ssl_sni_reneg.t
--- a/ssl_sni_reneg.t
+++ b/ssl_sni_reneg.t
@@ -41,7 +41,7 @@ http {
 
     ssl_certificate_key localhost.key;
     ssl_certificate localhost.crt;
-    ssl_protocols TLSv1.2;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
 
     server {
         listen       127.0.0.1:8443 ssl;

From mdounin at mdounin.ru  Sun Jul  9 21:24:56 2023
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Mon, 10 Jul 2023 00:24:56 +0300
Subject: [PATCH] Tests: enabled TLSv1 in uwsgi SSL tests
Message-ID: <cab7d252c0d3445eef3f.1688937896@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1688937878 -10800
#      Mon Jul 10 00:24:38 2023 +0300
# Node ID cab7d252c0d3445eef3f582d2879ac5fc3202019
# Parent  85188791cd9cf688a29401e31221551345b76ff4
Tests: enabled TLSv1 in uwsgi SSL tests.

In uWSGI starting with 2.0.17.1, TLSv1 is disabled by default.  It is
now re-enabled to make it possible to run tests with OpenSSL before 1.0.1,
where TLSv1.1 and TLSv1.2 support was introduced.

diff --git a/uwsgi_ssl.t b/uwsgi_ssl.t
--- a/uwsgi_ssl.t
+++ b/uwsgi_ssl.t
@@ -98,6 +98,11 @@ if ($uwsgihelp !~ /--wsgi-file/) {
 	push @uwsgiopts, '--plugin', 'python3';
 }
 
+if ($uwsgihelp =~ /--ssl-enable-tlsv1/) {
+	# uwsgi disables TLSv1 by default since 2.0.17.1
+	push @uwsgiopts, '--ssl-enable-tlsv1';
+}
+
 open OLDERR, ">&", \*STDERR; close STDERR;
 $t->run_daemon('uwsgi', @uwsgiopts,
 	'--ssl-socket', '127.0.0.1:' . port(8081) . ",$crt,$key",
diff --git a/uwsgi_ssl_verify.t b/uwsgi_ssl_verify.t
--- a/uwsgi_ssl_verify.t
+++ b/uwsgi_ssl_verify.t
@@ -144,6 +144,11 @@ if ($uwsgihelp !~ /--wsgi-file/) {
 	push @uwsgiopts, '--plugin', 'python3';
 }
 
+if ($uwsgihelp =~ /--ssl-enable-tlsv1/) {
+	# uwsgi disables TLSv1 by default since 2.0.17.1
+	push @uwsgiopts, '--ssl-enable-tlsv1';
+}
+
 open OLDERR, ">&", \*STDERR; close STDERR;
 $t->run_daemon('uwsgi', @uwsgiopts,
 	'--ssl-socket', '127.0.0.1:' . port(8081) . ",$crt1,$key1",

From pluknet at nginx.com  Mon Jul 10 16:23:42 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Mon, 10 Jul 2023 20:23:42 +0400
Subject: [PATCH 1 of 2] Tests: fixed h2_http2.t when nginx compiled
 without ALPN support
In-Reply-To: <bf9961a4507784b66965.1688170916@vm-bsd.mdounin.ru>
References: <patchbomb.1688170915@vm-bsd.mdounin.ru>
 <bf9961a4507784b66965.1688170916@vm-bsd.mdounin.ru>
Message-ID: <D743ADBC-768E-43F1-B246-D582C35517DA@nginx.com>


> On 1 Jul 2023, at 04:21, Maxim Dounin <mdounin at mdounin.ru> wrote:
> 
> # HG changeset patch
> # User Maxim Dounin <mdounin at mdounin.ru>
> # Date 1688164311 -10800
> #      Sat Jul 01 01:31:51 2023 +0300
> # Node ID bf9961a4507784b669650e7ef1d3cbacce8c94e1
> # Parent  22f45bf99a9e39e02d8ce07532d2cbfc89e7d8d1
> Tests: fixed h2_http2.t when nginx compiled without ALPN support.
> 
> ALPN support is only available with OpenSSL 1.0.2 or newer.
> 
> diff --git a/h2_http2.t b/h2_http2.t
> --- a/h2_http2.t
> +++ b/h2_http2.t
> @@ -127,9 +127,16 @@ ok(!get_ssl_socket(8443, 'disabled'), 's
> 
> }
> 
> +TODO: {
> +local $TODO = 'OpenSSL too old'
> +	if $t->has_module('OpenSSL')
> +	and not $t->has_feature('openssl:1.0.2');
> +
> is(get_https(8443, 'http2'), 200, 'host to enabled');
> is(get_https(8443, 'disabled', 'http2'), 421, 'host to disabled');
> 
> +}
> +
> # make sure HTTP/2 can be enabled selectively on virtual servers
> 

Looks good, tnx.

This got unnoticed since OpenSSL 1.0.1 no longer present in buildbot.
Personally I stopped using it in manual tests as it doesn't seem to be
correctly configured so cannot be built on Apple M1.

-- 
Sergey Kandaurov

From pluknet at nginx.com  Mon Jul 10 16:23:44 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Mon, 10 Jul 2023 20:23:44 +0400
Subject: [PATCH 2 of 2] Tests: adjusted TODO for OpenSSL 1.0.2h and up in
 h2_http2.t
In-Reply-To: <5ef10e454094ad5d2315.1688170917@vm-bsd.mdounin.ru>
References: <patchbomb.1688170915@vm-bsd.mdounin.ru>
 <5ef10e454094ad5d2315.1688170917@vm-bsd.mdounin.ru>
Message-ID: <7AC74D5E-8095-4D18-B303-6C5B1F6E20B3@nginx.com>


> On 1 Jul 2023, at 04:21, Maxim Dounin <mdounin at mdounin.ru> wrote:
> 
> # HG changeset patch
> # User Maxim Dounin <mdounin at mdounin.ru>
> # Date 1688164312 -10800
> #      Sat Jul 01 01:31:52 2023 +0300
> # Node ID 5ef10e454094ad5d231552f0fc2c386e9cc33585
> # Parent  bf9961a4507784b669650e7ef1d3cbacce8c94e1
> Tests: adjusted TODO for OpenSSL 1.0.2h and up in h2_http2.t.
> 
> OpenSSL uses correct SNI/ALPN callback order (SNI callback before ALPN
> callback) starting with OpenSSL 1.0.2h, so "sni to enabled" test
> is expected to succeed starting with OpenSSL 1.0.2h.
> 
> With this change, the "openssl:..." feature test now supports checking
> patch level encoded as letters, such as in "openssl:1.0.2h".
> 
> diff --git a/h2_http2.t b/h2_http2.t
> --- a/h2_http2.t
> +++ b/h2_http2.t
> @@ -151,9 +151,9 @@ ok(!get_ssl_socket(8444), 'default to di
> TODO: {
> local $TODO = 'broken ALPN/SNI order in LibreSSL'
> 	if $t->has_module('LibreSSL');
> -local $TODO = 'OpenSSL too old'
> +local $TODO = 'broken ALPN/SNI order in OpenSSL before 1.0.2h'
> 	if $t->has_module('OpenSSL')
> -	and not $t->has_feature('openssl:1.1.0');
> +	and not $t->has_feature('openssl:1.0.2h');
> 
> is(get_https(8444, 'http2'), 200, 'sni to enabled');
> 
> diff --git a/lib/Test/Nginx.pm b/lib/Test/Nginx.pm
> --- a/lib/Test/Nginx.pm
> +++ b/lib/Test/Nginx.pm
> @@ -266,20 +266,22 @@ sub has_feature($) {
> 		return 0;
> 	}
> 
> -	if ($feature =~ /^(openssl|libressl):([0-9.]+)/) {
> +	if ($feature =~ /^(openssl|libressl):([0-9.]+)([a-z]*)/) {
> 		my $library = $1;
> 		my $need = $2;
> +		my $patch = $3;
> 
> 		$self->{_configure_args} = `$NGINX -V 2>&1`
> 			if !defined $self->{_configure_args};
> 
> 		return 0 unless
> -			$self->{_configure_args} =~ /with $library ([0-9.]+)/i;
> +			$self->{_configure_args}
> +			=~ /with $library ([0-9.]+)([a-z]*)/i;

Note that patch versions with capital letters may now be captured
which cannot be specified in has_feature(), should never happen though.

> 
> -		my @v = split(/\./, $1);
> +		my @v = (split(/\./, $1), unpack("C*", $2));
> 		my ($n, $v);
> 
> -		for $n (split(/\./, $need)) {
> +		for $n (split(/\./, $need), unpack("C*", $patch)) {
> 			$v = shift @v || 0;
> 			return 0 if $n > $v;
> 			return 1 if $v > $n;

Looks good.

-- 
Sergey Kandaurov

From pluknet at nginx.com  Tue Jul 11 11:49:12 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Tue, 11 Jul 2023 15:49:12 +0400
Subject: [PATCH] Tests: enabled TLSv1 in ssl_sni_reneg.t
In-Reply-To: <85188791cd9cf688a294.1688937865@vm-bsd.mdounin.ru>
References: <85188791cd9cf688a294.1688937865@vm-bsd.mdounin.ru>
Message-ID: <38927C24-43FA-43CB-86A2-3192457639B8@nginx.com>


> On 10 Jul 2023, at 01:24, Maxim Dounin <mdounin at mdounin.ru> wrote:
> 
> # HG changeset patch
> # User Maxim Dounin <mdounin at mdounin.ru>
> # Date 1688521184 -10800
> #      Wed Jul 05 04:39:44 2023 +0300
> # Node ID 85188791cd9cf688a29401e31221551345b76ff4
> # Parent  c5767845481fc1d7df3e56b604fc4afdeab7be85
> Tests: enabled TLSv1 in ssl_sni_reneg.t.
> 
> This fixes running the test with OpenSSL before 1.0.1, where TLSv1.2
> support was introduced.
> 
> diff --git a/ssl_sni_reneg.t b/ssl_sni_reneg.t
> --- a/ssl_sni_reneg.t
> +++ b/ssl_sni_reneg.t
> @@ -41,7 +41,7 @@ http {
> 
>     ssl_certificate_key localhost.key;
>     ssl_certificate localhost.crt;
> -    ssl_protocols TLSv1.2;
> +    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
> 
>     server {
>         listen       127.0.0.1:8443 ssl;

Looks good per se.
Though it doesn't answer how to best handle this in general
after disabling TLSv1 and TLSv1.1 by default (see ticket #1911),
which means unusable default configurations with OpenSSL < 1.0.1.
Updating affected configurations to turn it back might be painful
(nginx-tests is a good such example).

Disabling TLSv1/TLSv1.1 is a reason to also drop support for old
OpenSSL versions before 1.0.1 altogether, that is 0.9.8 and 1.0.0.
(Another option might be to keep TLSv1/TLSv1.1 enabled by default
iff the library doesn't support TLSv1.2 and above.)

For the record, date of the last commit in OpenSSL git branches
and the corresponding date of branch support removal in nginx:

0.9.6   2005-05 2009-06 4y
0.9.7   2008-10 2016-04 7y6m
0.9.8   2015-12 n/a     7y+
1.0.0   2016-02 n/a     7y+

[+] still counting

-- 
Sergey Kandaurov

From pluknet at nginx.com  Tue Jul 11 11:49:50 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Tue, 11 Jul 2023 15:49:50 +0400
Subject: [PATCH] Tests: enabled TLSv1 in uwsgi SSL tests
In-Reply-To: <cab7d252c0d3445eef3f.1688937896@vm-bsd.mdounin.ru>
References: <cab7d252c0d3445eef3f.1688937896@vm-bsd.mdounin.ru>
Message-ID: <4CF7B3BA-9758-406C-AA29-96BE3CBEBCB5@nginx.com>


> On 10 Jul 2023, at 01:24, Maxim Dounin <mdounin at mdounin.ru> wrote:
> 
> # HG changeset patch
> # User Maxim Dounin <mdounin at mdounin.ru>
> # Date 1688937878 -10800
> #      Mon Jul 10 00:24:38 2023 +0300
> # Node ID cab7d252c0d3445eef3f582d2879ac5fc3202019
> # Parent  85188791cd9cf688a29401e31221551345b76ff4
> Tests: enabled TLSv1 in uwsgi SSL tests.
> 
> In uWSGI starting with 2.0.17.1, TLSv1 is disabled by default.  It is
> now re-enabled to make it possible to run tests with OpenSSL before 1.0.1,
> where TLSv1.1 and TLSv1.2 support was introduced.
> 
> diff --git a/uwsgi_ssl.t b/uwsgi_ssl.t
> --- a/uwsgi_ssl.t
> +++ b/uwsgi_ssl.t
> @@ -98,6 +98,11 @@ if ($uwsgihelp !~ /--wsgi-file/) {
> 	push @uwsgiopts, '--plugin', 'python3';
> }
> 
> +if ($uwsgihelp =~ /--ssl-enable-tlsv1/) {
> +	# uwsgi disables TLSv1 by default since 2.0.17.1
> +	push @uwsgiopts, '--ssl-enable-tlsv1';
> +}
> +
> open OLDERR, ">&", \*STDERR; close STDERR;
> $t->run_daemon('uwsgi', @uwsgiopts,
> 	'--ssl-socket', '127.0.0.1:' . port(8081) . ",$crt,$key",
> diff --git a/uwsgi_ssl_verify.t b/uwsgi_ssl_verify.t
> --- a/uwsgi_ssl_verify.t
> +++ b/uwsgi_ssl_verify.t
> @@ -144,6 +144,11 @@ if ($uwsgihelp !~ /--wsgi-file/) {
> 	push @uwsgiopts, '--plugin', 'python3';
> }
> 
> +if ($uwsgihelp =~ /--ssl-enable-tlsv1/) {
> +	# uwsgi disables TLSv1 by default since 2.0.17.1
> +	push @uwsgiopts, '--ssl-enable-tlsv1';
> +}
> +
> open OLDERR, ">&", \*STDERR; close STDERR;
> $t->run_daemon('uwsgi', @uwsgiopts,
> 	'--ssl-socket', '127.0.0.1:' . port(8081) . ",$crt1,$key1",

Looks good.

-- 
Sergey Kandaurov

From mdounin at mdounin.ru  Tue Jul 11 23:42:47 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Wed, 12 Jul 2023 02:42:47 +0300
Subject: [PATCH 1 of 2] Tests: fixed h2_http2.t when nginx compiled
 without ALPN support
In-Reply-To: <D743ADBC-768E-43F1-B246-D582C35517DA@nginx.com>
References: <patchbomb.1688170915@vm-bsd.mdounin.ru>
 <bf9961a4507784b66965.1688170916@vm-bsd.mdounin.ru>
 <D743ADBC-768E-43F1-B246-D582C35517DA@nginx.com>
Message-ID: <ZK3o96ooehbbaaD5@mdounin.ru>

Hello!

On Mon, Jul 10, 2023 at 08:23:42PM +0400, Sergey Kandaurov wrote:

> > On 1 Jul 2023, at 04:21, Maxim Dounin <mdounin at mdounin.ru> wrote:
> > 
> > # HG changeset patch
> > # User Maxim Dounin <mdounin at mdounin.ru>
> > # Date 1688164311 -10800
> > #      Sat Jul 01 01:31:51 2023 +0300
> > # Node ID bf9961a4507784b669650e7ef1d3cbacce8c94e1
> > # Parent  22f45bf99a9e39e02d8ce07532d2cbfc89e7d8d1
> > Tests: fixed h2_http2.t when nginx compiled without ALPN support.
> > 
> > ALPN support is only available with OpenSSL 1.0.2 or newer.
> > 
> > diff --git a/h2_http2.t b/h2_http2.t
> > --- a/h2_http2.t
> > +++ b/h2_http2.t
> > @@ -127,9 +127,16 @@ ok(!get_ssl_socket(8443, 'disabled'), 's
> > 
> > }
> > 
> > +TODO: {
> > +local $TODO = 'OpenSSL too old'
> > +	if $t->has_module('OpenSSL')
> > +	and not $t->has_feature('openssl:1.0.2');
> > +
> > is(get_https(8443, 'http2'), 200, 'host to enabled');
> > is(get_https(8443, 'disabled', 'http2'), 421, 'host to disabled');
> > 
> > +}
> > +
> > # make sure HTTP/2 can be enabled selectively on virtual servers
> > 
> 
> Looks good, tnx.

Pushed to http://mdounin.ru/hg/nginx-tests, thanks.

> This got unnoticed since OpenSSL 1.0.1 no longer present in buildbot.
> Personally I stopped using it in manual tests as it doesn't seem to be
> correctly configured so cannot be built on Apple M1.

I usually test non-trivial SSL-related changes with all supported 
OpenSSL branches, hence it was caught.

-- 
Maxim Dounin
http://mdounin.ru/

From mdounin at mdounin.ru  Tue Jul 11 23:42:57 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Wed, 12 Jul 2023 02:42:57 +0300
Subject: [PATCH 2 of 2] Tests: adjusted TODO for OpenSSL 1.0.2h and up in
 h2_http2.t
In-Reply-To: <7AC74D5E-8095-4D18-B303-6C5B1F6E20B3@nginx.com>
References: <patchbomb.1688170915@vm-bsd.mdounin.ru>
 <5ef10e454094ad5d2315.1688170917@vm-bsd.mdounin.ru>
 <7AC74D5E-8095-4D18-B303-6C5B1F6E20B3@nginx.com>
Message-ID: <ZK3pAcF6KSlycwrI@mdounin.ru>

Hello!

On Mon, Jul 10, 2023 at 08:23:44PM +0400, Sergey Kandaurov wrote:

> > On 1 Jul 2023, at 04:21, Maxim Dounin <mdounin at mdounin.ru> wrote:
> > 
> > # HG changeset patch
> > # User Maxim Dounin <mdounin at mdounin.ru>
> > # Date 1688164312 -10800
> > #      Sat Jul 01 01:31:52 2023 +0300
> > # Node ID 5ef10e454094ad5d231552f0fc2c386e9cc33585
> > # Parent  bf9961a4507784b669650e7ef1d3cbacce8c94e1
> > Tests: adjusted TODO for OpenSSL 1.0.2h and up in h2_http2.t.
> > 
> > OpenSSL uses correct SNI/ALPN callback order (SNI callback before ALPN
> > callback) starting with OpenSSL 1.0.2h, so "sni to enabled" test
> > is expected to succeed starting with OpenSSL 1.0.2h.
> > 
> > With this change, the "openssl:..." feature test now supports checking
> > patch level encoded as letters, such as in "openssl:1.0.2h".
> > 
> > diff --git a/h2_http2.t b/h2_http2.t
> > --- a/h2_http2.t
> > +++ b/h2_http2.t
> > @@ -151,9 +151,9 @@ ok(!get_ssl_socket(8444), 'default to di
> > TODO: {
> > local $TODO = 'broken ALPN/SNI order in LibreSSL'
> > 	if $t->has_module('LibreSSL');
> > -local $TODO = 'OpenSSL too old'
> > +local $TODO = 'broken ALPN/SNI order in OpenSSL before 1.0.2h'
> > 	if $t->has_module('OpenSSL')
> > -	and not $t->has_feature('openssl:1.1.0');
> > +	and not $t->has_feature('openssl:1.0.2h');
> > 
> > is(get_https(8444, 'http2'), 200, 'sni to enabled');
> > 
> > diff --git a/lib/Test/Nginx.pm b/lib/Test/Nginx.pm
> > --- a/lib/Test/Nginx.pm
> > +++ b/lib/Test/Nginx.pm
> > @@ -266,20 +266,22 @@ sub has_feature($) {
> > 		return 0;
> > 	}
> > 
> > -	if ($feature =~ /^(openssl|libressl):([0-9.]+)/) {
> > +	if ($feature =~ /^(openssl|libressl):([0-9.]+)([a-z]*)/) {
> > 		my $library = $1;
> > 		my $need = $2;
> > +		my $patch = $3;
> > 
> > 		$self->{_configure_args} = `$NGINX -V 2>&1`
> > 			if !defined $self->{_configure_args};
> > 
> > 		return 0 unless
> > -			$self->{_configure_args} =~ /with $library ([0-9.]+)/i;
> > +			$self->{_configure_args}
> > +			=~ /with $library ([0-9.]+)([a-z]*)/i;
> 
> Note that patch versions with capital letters may now be captured
> which cannot be specified in has_feature(), should never happen though.

I don't think it matters.  And, even if some library will report 
patch level with capital letters, there will be no practical 
difference anyway, as capital letters will be interpreted 
identically to no letters at all.

> > 
> > -		my @v = split(/\./, $1);
> > +		my @v = (split(/\./, $1), unpack("C*", $2));
> > 		my ($n, $v);
> > 
> > -		for $n (split(/\./, $need)) {
> > +		for $n (split(/\./, $need), unpack("C*", $patch)) {
> > 			$v = shift @v || 0;
> > 			return 0 if $n > $v;
> > 			return 1 if $v > $n;
> 
> Looks good.

Pushed to http://mdounin.ru/hg/nginx-tests, thanks.

-- 
Maxim Dounin
http://mdounin.ru/

From mdounin at mdounin.ru  Tue Jul 11 23:43:10 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Wed, 12 Jul 2023 02:43:10 +0300
Subject: [PATCH] Tests: enabled TLSv1 in uwsgi SSL tests
In-Reply-To: <4CF7B3BA-9758-406C-AA29-96BE3CBEBCB5@nginx.com>
References: <cab7d252c0d3445eef3f.1688937896@vm-bsd.mdounin.ru>
 <4CF7B3BA-9758-406C-AA29-96BE3CBEBCB5@nginx.com>
Message-ID: <ZK3pDjxSbyeVOqtn@mdounin.ru>

Hello!

On Tue, Jul 11, 2023 at 03:49:50PM +0400, Sergey Kandaurov wrote:

> > On 10 Jul 2023, at 01:24, Maxim Dounin <mdounin at mdounin.ru> wrote:
> > 
> > # HG changeset patch
> > # User Maxim Dounin <mdounin at mdounin.ru>
> > # Date 1688937878 -10800
> > #      Mon Jul 10 00:24:38 2023 +0300
> > # Node ID cab7d252c0d3445eef3f582d2879ac5fc3202019
> > # Parent  85188791cd9cf688a29401e31221551345b76ff4
> > Tests: enabled TLSv1 in uwsgi SSL tests.
> > 
> > In uWSGI starting with 2.0.17.1, TLSv1 is disabled by default.  It is
> > now re-enabled to make it possible to run tests with OpenSSL before 1.0.1,
> > where TLSv1.1 and TLSv1.2 support was introduced.
> > 
> > diff --git a/uwsgi_ssl.t b/uwsgi_ssl.t
> > --- a/uwsgi_ssl.t
> > +++ b/uwsgi_ssl.t
> > @@ -98,6 +98,11 @@ if ($uwsgihelp !~ /--wsgi-file/) {
> > 	push @uwsgiopts, '--plugin', 'python3';
> > }
> > 
> > +if ($uwsgihelp =~ /--ssl-enable-tlsv1/) {
> > +	# uwsgi disables TLSv1 by default since 2.0.17.1
> > +	push @uwsgiopts, '--ssl-enable-tlsv1';
> > +}
> > +
> > open OLDERR, ">&", \*STDERR; close STDERR;
> > $t->run_daemon('uwsgi', @uwsgiopts,
> > 	'--ssl-socket', '127.0.0.1:' . port(8081) . ",$crt,$key",
> > diff --git a/uwsgi_ssl_verify.t b/uwsgi_ssl_verify.t
> > --- a/uwsgi_ssl_verify.t
> > +++ b/uwsgi_ssl_verify.t
> > @@ -144,6 +144,11 @@ if ($uwsgihelp !~ /--wsgi-file/) {
> > 	push @uwsgiopts, '--plugin', 'python3';
> > }
> > 
> > +if ($uwsgihelp =~ /--ssl-enable-tlsv1/) {
> > +	# uwsgi disables TLSv1 by default since 2.0.17.1
> > +	push @uwsgiopts, '--ssl-enable-tlsv1';
> > +}
> > +
> > open OLDERR, ">&", \*STDERR; close STDERR;
> > $t->run_daemon('uwsgi', @uwsgiopts,
> > 	'--ssl-socket', '127.0.0.1:' . port(8081) . ",$crt1,$key1",
> 
> Looks good.

Pushed to http://mdounin.ru/hg/nginx-tests, thanks.

-- 
Maxim Dounin
http://mdounin.ru/

From mdounin at mdounin.ru  Tue Jul 11 23:43:55 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Wed, 12 Jul 2023 02:43:55 +0300
Subject: [PATCH] Tests: enabled TLSv1 in ssl_sni_reneg.t
In-Reply-To: <38927C24-43FA-43CB-86A2-3192457639B8@nginx.com>
References: <85188791cd9cf688a294.1688937865@vm-bsd.mdounin.ru>
 <38927C24-43FA-43CB-86A2-3192457639B8@nginx.com>
Message-ID: <ZK3pO4vXqkARHBXX@mdounin.ru>

Hello!

On Tue, Jul 11, 2023 at 03:49:12PM +0400, Sergey Kandaurov wrote:

> > On 10 Jul 2023, at 01:24, Maxim Dounin <mdounin at mdounin.ru> wrote:
> > 
> > # HG changeset patch
> > # User Maxim Dounin <mdounin at mdounin.ru>
> > # Date 1688521184 -10800
> > #      Wed Jul 05 04:39:44 2023 +0300
> > # Node ID 85188791cd9cf688a29401e31221551345b76ff4
> > # Parent  c5767845481fc1d7df3e56b604fc4afdeab7be85
> > Tests: enabled TLSv1 in ssl_sni_reneg.t.
> > 
> > This fixes running the test with OpenSSL before 1.0.1, where TLSv1.2
> > support was introduced.
> > 
> > diff --git a/ssl_sni_reneg.t b/ssl_sni_reneg.t
> > --- a/ssl_sni_reneg.t
> > +++ b/ssl_sni_reneg.t
> > @@ -41,7 +41,7 @@ http {
> > 
> >     ssl_certificate_key localhost.key;
> >     ssl_certificate localhost.crt;
> > -    ssl_protocols TLSv1.2;
> > +    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
> > 
> >     server {
> >         listen       127.0.0.1:8443 ssl;
> 
> Looks good per se.

Pushed to http://mdounin.ru/hg/nginx-tests, thanks.

> Though it doesn't answer how to best handle this in general
> after disabling TLSv1 and TLSv1.1 by default (see ticket #1911),
> which means unusable default configurations with OpenSSL < 1.0.1.
> Updating affected configurations to turn it back might be painful
> (nginx-tests is a good such example).

Yep, that's something to consider if/when we'll decide to disable 
TLSv1 and TLSv1.1 by default.

> Disabling TLSv1/TLSv1.1 is a reason to also drop support for old
> OpenSSL versions before 1.0.1 altogether, that is 0.9.8 and 1.0.0.
> (Another option might be to keep TLSv1/TLSv1.1 enabled by default
> iff the library doesn't support TLSv1.2 and above.)
> 
> For the record, date of the last commit in OpenSSL git branches
> and the corresponding date of branch support removal in nginx:
> 
> 0.9.6   2005-05 2009-06 4y
> 0.9.7   2008-10 2016-04 7y6m
> 0.9.8   2015-12 n/a     7y+
> 1.0.0   2016-02 n/a     7y+
> 
> [+] still counting

I tend to think we can more or less safely drop anything below 
OpenSSL 1.0.2 now: last release on the OpenSSL 1.0.1 branch, 
OpenSSL 1.0.1u, was at 2016-09-22, so in terms of dates it's not 
really different from the OpenSSL 1.0.0 branch.  Also, OpenSSL 
releases from the 1.0.1 branch are no longer present in any 
supported OSes I'm aware of.

On the other hand, support for OpenSSL at least back to 0.9.8zh 
does not seem to be a major issue.

-- 
Maxim Dounin
http://mdounin.ru/

From xeioex at nginx.com  Wed Jul 12 05:51:53 2023
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Wed, 12 Jul 2023 05:51:53 +0000
Subject: [njs] Version bump.
Message-ID: <hg.ac56314cc3be.1689141113.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/ac56314cc3be
branches:  
changeset: 2179:ac56314cc3be
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Mon Jul 10 17:57:45 2023 -0700
description:
Version bump.

diffstat:

 src/njs.h |  4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diffs (14 lines):

diff -r 034ad86967b6 -r ac56314cc3be src/njs.h
--- a/src/njs.h	Thu Jul 06 09:10:47 2023 -0700
+++ b/src/njs.h	Mon Jul 10 17:57:45 2023 -0700
@@ -11,8 +11,8 @@
 
 #include <njs_auto_config.h>
 
-#define NJS_VERSION                 "0.8.0"
-#define NJS_VERSION_NUMBER          0x000800
+#define NJS_VERSION                 "0.8.1"
+#define NJS_VERSION_NUMBER          0x000801
 
 
 #include <string.h>

From xeioex at nginx.com  Wed Jul 12 05:51:55 2023
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Wed, 12 Jul 2023 05:51:55 +0000
Subject: [njs] HTTP: fixed setting of Date header.
Message-ID: <hg.a35035c52201.1689141115.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/a35035c52201
branches:  
changeset: 2180:a35035c52201
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 11 19:12:34 2023 -0700
description:
HTTP: fixed setting of Date header.

Previously, r.headersOut['Date'] setter did not update
r->headers_out.date. As a result a client might get two
Date headers.

diffstat:

 nginx/ngx_http_js_module.c |  36 ++++++++++++++++++++++++++++++++++++
 nginx/t/js_headers.t       |  22 ++++++++++++++++++++--
 2 files changed, 56 insertions(+), 2 deletions(-)

diffs (138 lines):

diff -r ac56314cc3be -r a35035c52201 nginx/ngx_http_js_module.c
--- a/nginx/ngx_http_js_module.c	Mon Jul 10 17:57:45 2023 -0700
+++ b/nginx/ngx_http_js_module.c	Tue Jul 11 19:12:34 2023 -0700
@@ -125,6 +125,9 @@ static njs_int_t ngx_http_js_content_len
 static njs_int_t ngx_http_js_content_type122(njs_vm_t *vm,
     ngx_http_request_t *r, ngx_list_t *headers, njs_str_t *name,
     njs_value_t *setval, njs_value_t *retval);
+static njs_int_t ngx_http_js_date122(njs_vm_t *vm, ngx_http_request_t *r,
+    ngx_list_t *headers, njs_str_t *name, njs_value_t *setval,
+    njs_value_t *retval);
 static njs_int_t ngx_http_js_location122(njs_vm_t *vm, ngx_http_request_t *r,
     ngx_list_t *headers, njs_str_t *name, njs_value_t *setval,
     njs_value_t *retval);
@@ -222,6 +225,9 @@ static njs_int_t ngx_http_js_content_len
 static njs_int_t ngx_http_js_content_type(njs_vm_t *vm, ngx_http_request_t *r,
     unsigned flags, njs_str_t *name, njs_value_t *setval,
     njs_value_t *retval);
+static njs_int_t ngx_http_js_date(njs_vm_t *vm, ngx_http_request_t *r,
+    unsigned flags, njs_str_t *name, njs_value_t *setval,
+    njs_value_t *retval);
 static njs_int_t ngx_http_js_location(njs_vm_t *vm, ngx_http_request_t *r,
     unsigned flags, njs_str_t *name, njs_value_t *setval,
     njs_value_t *retval);
@@ -1526,6 +1532,7 @@ ngx_http_js_ext_header_out(njs_vm_t *vm,
         { njs_str("Content-Type"), ngx_http_js_content_type122 },
         { njs_str("Content-Length"), ngx_http_js_content_length122 },
         { njs_str("Content-Encoding"), ngx_http_js_content_encoding122 },
+        { njs_str("Date"), ngx_http_js_date122 },
         { njs_str("Etag"), ngx_http_js_header_single },
         { njs_str("Expires"), ngx_http_js_header_single },
         { njs_str("Last-Modified"), ngx_http_js_header_single },
@@ -1538,6 +1545,7 @@ ngx_http_js_ext_header_out(njs_vm_t *vm,
         { njs_str("Content-Encoding"), 0, ngx_http_js_content_encoding },
         { njs_str("Content-Length"), 0, ngx_http_js_content_length },
         { njs_str("Content-Type"), 0, ngx_http_js_content_type },
+        { njs_str("Date"), 0, ngx_http_js_date },
         { njs_str("Etag"), NJS_HEADER_SINGLE, ngx_http_js_header_out },
         { njs_str("Expires"), NJS_HEADER_SINGLE, ngx_http_js_header_out },
         { njs_str("Last-Modified"), NJS_HEADER_SINGLE, ngx_http_js_header_out },
@@ -1955,6 +1963,14 @@ ngx_http_js_content_type122(njs_vm_t *vm
 
 
 static njs_int_t
+ngx_http_js_date122(njs_vm_t *vm, ngx_http_request_t *r,
+    ngx_list_t *headers, njs_str_t *v, njs_value_t *setval, njs_value_t *retval)
+{
+    return ngx_http_js_date(vm, r, 0, v, setval, retval);
+}
+
+
+static njs_int_t
 ngx_http_js_location122(njs_vm_t *vm, ngx_http_request_t *r,
     ngx_list_t *headers, njs_str_t *v, njs_value_t *setval, njs_value_t *retval)
 {
@@ -3961,6 +3977,26 @@ ngx_http_js_content_type(njs_vm_t *vm, n
 
 
 static njs_int_t
+ngx_http_js_date(njs_vm_t *vm, ngx_http_request_t *r, unsigned flags,
+    njs_str_t *v, njs_value_t *setval, njs_value_t *retval)
+{
+    njs_int_t         rc;
+    ngx_table_elt_t  *h;
+
+    rc = ngx_http_js_header_out_special(vm, r, v, setval, retval, &h);
+    if (rc == NJS_ERROR) {
+        return NJS_ERROR;
+    }
+
+    if (setval != NULL || retval == NULL) {
+        r->headers_out.date = h;
+    }
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
 ngx_http_js_location(njs_vm_t *vm, ngx_http_request_t *r, unsigned flags,
     njs_str_t *v, njs_value_t *setval, njs_value_t *retval)
 {
diff -r ac56314cc3be -r a35035c52201 nginx/t/js_headers.t
--- a/nginx/t/js_headers.t	Mon Jul 10 17:57:45 2023 -0700
+++ b/nginx/t/js_headers.t	Tue Jul 11 19:12:34 2023 -0700
@@ -84,6 +84,10 @@ http {
             js_content test.content_encoding_arr;
         }
 
+        location /date {
+            js_content test.date;
+        }
+
         location /location {
             js_content test.location;
         }
@@ -245,6 +249,11 @@ EOF
         r.return(200);
     }
 
+    function date(r) {
+        r.headersOut['Date'] = 'Sun, 09 Sep 2001 01:46:40 GMT';
+        r.return(200);
+    }
+
     function location(r) {
         if (njs.version_number >= 0x000705) {
             var lc = r.headersOut['Location'];
@@ -437,12 +446,13 @@ EOF
                     hdr_in, raw_hdr_in, hdr_sorted_keys, foo_in, ifoo_in,
                     hdr_out, raw_hdr_out, hdr_out_array, hdr_out_single,
                     hdr_out_set_cookie, ihdr_out, hdr_out_special_set,
-                    copy_subrequest_hdrs, subrequest, location, location_sr};
+                    copy_subrequest_hdrs, subrequest, date, location,
+                    location_sr};
 
 
 EOF
 
-$t->try_run('no njs')->plan(44);
+$t->try_run('no njs')->plan(45);
 
 ###############################################################################
 
@@ -590,6 +600,14 @@ unlike(http_get('/location_sr'), qr/Loca
 
 }
 
+TODO: {
+local $TODO = 'not yet' unless has_version('0.8.1');
+
+like(http_get('/date'), qr/Date: Sun, 09 Sep 2001 01:46:40 GMT/,
+	'set date');
+
+}
+
 ###############################################################################
 
 sub has_version {

From xeioex at nginx.com  Wed Jul 12 05:51:57 2023
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Wed, 12 Jul 2023 05:51:57 +0000
Subject: [njs] HTTP: fixed setting of Last-Modified header.
Message-ID: <hg.3af6c729b7cb.1689141117.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/3af6c729b7cb
branches:  
changeset: 2181:3af6c729b7cb
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 11 22:50:44 2023 -0700
description:
HTTP: fixed setting of Last-Modified header.

Previously, r.headersOut['Last-Modified'] setter did not update
r->headers_out.last_modified. As a result a client might get two
Last-Modified headers.

diffstat:

 nginx/ngx_http_js_module.c |  38 ++++++++++++++++++++++++++++++++++++--
 nginx/t/js_headers.t       |  18 +++++++++++++++---
 2 files changed, 51 insertions(+), 5 deletions(-)

diffs (136 lines):

diff -r a35035c52201 -r 3af6c729b7cb nginx/ngx_http_js_module.c
--- a/nginx/ngx_http_js_module.c	Tue Jul 11 19:12:34 2023 -0700
+++ b/nginx/ngx_http_js_module.c	Tue Jul 11 22:50:44 2023 -0700
@@ -128,6 +128,9 @@ static njs_int_t ngx_http_js_content_typ
 static njs_int_t ngx_http_js_date122(njs_vm_t *vm, ngx_http_request_t *r,
     ngx_list_t *headers, njs_str_t *name, njs_value_t *setval,
     njs_value_t *retval);
+static njs_int_t ngx_http_js_last_modified122(njs_vm_t *vm,
+    ngx_http_request_t *r, ngx_list_t *headers, njs_str_t *name,
+    njs_value_t *setval, njs_value_t *retval);
 static njs_int_t ngx_http_js_location122(njs_vm_t *vm, ngx_http_request_t *r,
     ngx_list_t *headers, njs_str_t *name, njs_value_t *setval,
     njs_value_t *retval);
@@ -228,6 +231,9 @@ static njs_int_t ngx_http_js_content_typ
 static njs_int_t ngx_http_js_date(njs_vm_t *vm, ngx_http_request_t *r,
     unsigned flags, njs_str_t *name, njs_value_t *setval,
     njs_value_t *retval);
+static njs_int_t ngx_http_js_last_modified(njs_vm_t *vm, ngx_http_request_t *r,
+    unsigned flags, njs_str_t *name, njs_value_t *setval,
+    njs_value_t *retval);
 static njs_int_t ngx_http_js_location(njs_vm_t *vm, ngx_http_request_t *r,
     unsigned flags, njs_str_t *name, njs_value_t *setval,
     njs_value_t *retval);
@@ -1535,7 +1541,7 @@ ngx_http_js_ext_header_out(njs_vm_t *vm,
         { njs_str("Date"), ngx_http_js_date122 },
         { njs_str("Etag"), ngx_http_js_header_single },
         { njs_str("Expires"), ngx_http_js_header_single },
-        { njs_str("Last-Modified"), ngx_http_js_header_single },
+        { njs_str("Last-Modified"), ngx_http_js_last_modified122 },
         { njs_str("Location"), ngx_http_js_location122 },
         { njs_str("Set-Cookie"), ngx_http_js_header_array },
         { njs_str("Retry-After"), ngx_http_js_header_single },
@@ -1548,7 +1554,7 @@ ngx_http_js_ext_header_out(njs_vm_t *vm,
         { njs_str("Date"), 0, ngx_http_js_date },
         { njs_str("Etag"), NJS_HEADER_SINGLE, ngx_http_js_header_out },
         { njs_str("Expires"), NJS_HEADER_SINGLE, ngx_http_js_header_out },
-        { njs_str("Last-Modified"), NJS_HEADER_SINGLE, ngx_http_js_header_out },
+        { njs_str("Last-Modified"), 0, ngx_http_js_last_modified },
         { njs_str("Location"), 0, ngx_http_js_location },
         { njs_str("Set-Cookie"), NJS_HEADER_ARRAY, ngx_http_js_header_out },
         { njs_str("Retry-After"), NJS_HEADER_SINGLE, ngx_http_js_header_out },
@@ -1971,6 +1977,14 @@ ngx_http_js_date122(njs_vm_t *vm, ngx_ht
 
 
 static njs_int_t
+ngx_http_js_last_modified122(njs_vm_t *vm, ngx_http_request_t *r,
+    ngx_list_t *headers, njs_str_t *v, njs_value_t *setval, njs_value_t *retval)
+{
+    return ngx_http_js_last_modified(vm, r, 0, v, setval, retval);
+}
+
+
+static njs_int_t
 ngx_http_js_location122(njs_vm_t *vm, ngx_http_request_t *r,
     ngx_list_t *headers, njs_str_t *v, njs_value_t *setval, njs_value_t *retval)
 {
@@ -3997,6 +4011,26 @@ ngx_http_js_date(njs_vm_t *vm, ngx_http_
 
 
 static njs_int_t
+ngx_http_js_last_modified(njs_vm_t *vm, ngx_http_request_t *r, unsigned flags,
+    njs_str_t *v, njs_value_t *setval, njs_value_t *retval)
+{
+    njs_int_t         rc;
+    ngx_table_elt_t  *h;
+
+    rc = ngx_http_js_header_out_special(vm, r, v, setval, retval, &h);
+    if (rc == NJS_ERROR) {
+        return NJS_ERROR;
+    }
+
+    if (setval != NULL || retval == NULL) {
+        r->headers_out.last_modified = h;
+    }
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
 ngx_http_js_location(njs_vm_t *vm, ngx_http_request_t *r, unsigned flags,
     njs_str_t *v, njs_value_t *setval, njs_value_t *retval)
 {
diff -r a35035c52201 -r 3af6c729b7cb nginx/t/js_headers.t
--- a/nginx/t/js_headers.t	Tue Jul 11 19:12:34 2023 -0700
+++ b/nginx/t/js_headers.t	Tue Jul 11 22:50:44 2023 -0700
@@ -88,6 +88,10 @@ http {
             js_content test.date;
         }
 
+        location /last_modified {
+            js_content test.last_modified;
+        }
+
         location /location {
             js_content test.location;
         }
@@ -254,6 +258,11 @@ EOF
         r.return(200);
     }
 
+    function last_modified(r) {
+        r.headersOut['Last-Modified'] = 'Sun, 09 Sep 2001 01:46:40 GMT';
+        r.return(200);
+    }
+
     function location(r) {
         if (njs.version_number >= 0x000705) {
             var lc = r.headersOut['Location'];
@@ -446,13 +455,13 @@ EOF
                     hdr_in, raw_hdr_in, hdr_sorted_keys, foo_in, ifoo_in,
                     hdr_out, raw_hdr_out, hdr_out_array, hdr_out_single,
                     hdr_out_set_cookie, ihdr_out, hdr_out_special_set,
-                    copy_subrequest_hdrs, subrequest, date, location,
-                    location_sr};
+                    copy_subrequest_hdrs, subrequest, date, last_modified,
+                    location, location_sr};
 
 
 EOF
 
-$t->try_run('no njs')->plan(45);
+$t->try_run('no njs')->plan(46);
 
 ###############################################################################
 
@@ -605,6 +614,9 @@ local $TODO = 'not yet' unless has_versi
 
 like(http_get('/date'), qr/Date: Sun, 09 Sep 2001 01:46:40 GMT/,
 	'set date');
+like(http_get('/last_modified'),
+	qr/Last-Modified: Sun, 09 Sep 2001 01:46:40 GMT/,
+	'set Last-Modified');
 
 }
 

From pluknet at nginx.com  Wed Jul 12 11:33:16 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Wed, 12 Jul 2023 11:33:16 +0000
Subject: [nginx] HTTP/3: fixed $body_bytes_sent.
Message-ID: <hg.f91dc350be9f.1689161596.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/f91dc350be9f
branches:  
changeset: 9133:f91dc350be9f
user:      Sergey Kandaurov <pluknet at nginx.com>
date:      Wed Jul 12 15:27:35 2023 +0400
description:
HTTP/3: fixed $body_bytes_sent.

diffstat:

 src/http/v3/ngx_http_v3_filter_module.c |  1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diffs (11 lines):

diff -r 77c1418916f7 -r f91dc350be9f src/http/v3/ngx_http_v3_filter_module.c
--- a/src/http/v3/ngx_http_v3_filter_module.c	Thu Jun 08 14:58:01 2023 +0400
+++ b/src/http/v3/ngx_http_v3_filter_module.c	Wed Jul 12 15:27:35 2023 +0400
@@ -581,6 +581,7 @@ ngx_http_v3_header_filter(ngx_http_reque
 
     for (cl = out; cl; cl = cl->next) {
         h3c->total_bytes += cl->buf->last - cl->buf->pos;
+        r->header_size += cl->buf->last - cl->buf->pos;
     }
 
     return ngx_http_write_filter(r, out);

From vesa.jaaskelainen at vaisala.com  Wed Jul 12 14:07:03 2023
From: vesa.jaaskelainen at vaisala.com (=?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=)
Date: Wed, 12 Jul 2023 17:07:03 +0300
Subject: [PATCH 0/4] SSL: Add support for loading X.509 certificates from
 openssl engine
Message-ID: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>

(I hope this goes properly out as I had major issues with hg email so
combined hg export + git send-email)

It is convenient to keep X.509 certificates related to key pairs stored in
openssl engine within the engine.

Implementation uses 'LOAD_CERT_CTRL' extension to fetch certificate from
the engine. This extension is not supported by all engines and in those
cases it should report with an error.

Configuration is similar to what it is for 'ssl_certificate_key'.

First certificate must match with ssl_certificate_key's key pair rest of
the certificiates are added to the certificate chain.

Example configuration with libp11's pkcs11 engine:

  ssl_certificate      "engine:pkcs11:pkcs11:token=mytoken;object=mykey
                        engine:pkcs11:pkcs11:token=mytoken;object=int-ca";
  ssl_certificate_key  "engine:pkcs11:pkcs11:token=mytoken;object=mykey?pin-value=mypin";

Tested the loading with two pkcs11 implementations SoftHSMv2 and with
OP-TEE's PKCS11 Trusted Application running on Embedded Linux device.

First three commits is the main beef and in order to make it more flexible
added also last commit allowing intermediate certificates loaded from file
system.

Separator of space is used as there was already existing use of array for
ssl_certificate configuration.

Thanks,
Vesa Jääskeläinen

From vesa.jaaskelainen at vaisala.com  Wed Jul 12 14:07:04 2023
From: vesa.jaaskelainen at vaisala.com (=?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=)
Date: Wed, 12 Jul 2023 17:07:04 +0300
Subject: [PATCH 1/4] SSL: Add support for loading X.509 certificate from
 openssl engine
In-Reply-To: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
References: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
Message-ID: <20230712140707.364629-2-vesa.jaaskelainen@vaisala.com>

# HG changeset patch
# User Vesa Jääskeläinen <vesa.jaaskelainen at vaisala.com>
# Date 1689169151 -10800
#      Wed Jul 12 16:39:11 2023 +0300
# Node ID 57f9cc8ef8ce24385ebb7bb4cf3150299bc8ed91
# Parent  f91dc350be9f4a6bf1379a32a210afece7b0d75e
SSL: Add support for loading X.509 certificate from openssl engine

It is convenient to keep X.509 certificates related to key pairs stored in
openssl engine within the engine.

Implementation uses 'LOAD_CERT_CTRL' extension to fetch certificate from
the engine. This extension is not supported by all engines and in those
cases it should report with an error.

Configuration is similar to what it is for 'ssl_certificate_key'.

Example configuration with libp11's pkcs11 engine:

  ssl_certificate      "engine:pkcs11:pkcs11:token=mytoken;object=mykey";
  ssl_certificate_key  "engine:pkcs11:pkcs11:token=mytoken;object=mykey?pin-value=mypin";

diff -r f91dc350be9f -r 57f9cc8ef8ce src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c	Wed Jul 12 15:27:35 2023 +0400
+++ b/src/event/ngx_event_openssl.c	Wed Jul 12 16:39:11 2023 +0300
@@ -607,6 +607,24 @@
     return NGX_OK;
 }
 
+static X509 *
+ngx_ssl_engine_load_certificate(ENGINE *engine, const char * cert_id)
+{
+    struct {
+        const char * cert_id;
+        X509 * cert;
+    } load_cert;
+
+    load_cert.cert_id = cert_id;
+    load_cert.cert = NULL;
+
+    /* Note: if certificate is found -- X509 object's ownership is transferred. */
+    if (!ENGINE_ctrl_cmd(engine, "LOAD_CERT_CTRL", 0, &load_cert, NULL, 0)) {
+        return NULL;
+    }
+
+    return load_cert.cert;
+}
 
 static X509 *
 ngx_ssl_load_certificate(ngx_pool_t *pool, char **err, ngx_str_t *cert,
@@ -616,6 +634,50 @@
     X509    *x509, *temp;
     u_long   n;
 
+    if (ngx_strncmp(cert->data, "engine:", sizeof("engine:") - 1) == 0) {
+
+#ifndef OPENSSL_NO_ENGINE
+
+        u_char  *p, *last;
+        ENGINE  *engine;
+
+        p = cert->data + sizeof("engine:") - 1;
+        last = (u_char *) ngx_strchr(p, ':');
+
+        if (last == NULL) {
+            *err = "invalid syntax";
+            return NULL;
+        }
+
+        *last = '\0';
+
+        engine = ENGINE_by_id((char *) p);
+
+        if (engine == NULL) {
+            *err = "ENGINE_by_id() failed";
+            return NULL;
+        }
+
+        *last++ = ':';
+
+        x509 = ngx_ssl_engine_load_certificate(engine, (char *) last);
+
+        if (x509 == NULL) {
+            *err = "ngx_ssl_engine_load_certificate() failed";
+            return NULL;
+        }
+
+        ENGINE_free(engine);
+        return x509;
+
+#else
+
+        *err = "loading \"engine:...\" certificate is not supported";
+        return NULL;
+
+#endif
+    }
+
     if (ngx_strncmp(cert->data, "data:", sizeof("data:") - 1) == 0) {
 
         bio = BIO_new_mem_buf(cert->data + sizeof("data:") - 1,

From vesa.jaaskelainen at vaisala.com  Wed Jul 12 14:07:05 2023
From: vesa.jaaskelainen at vaisala.com (=?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=)
Date: Wed, 12 Jul 2023 17:07:05 +0300
Subject: [PATCH 2/4] Core: Add ngx_strtok_r()
In-Reply-To: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
References: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
Message-ID: <20230712140707.364629-3-vesa.jaaskelainen@vaisala.com>

# HG changeset patch
# User Vesa Jääskeläinen <vesa.jaaskelainen at vaisala.com>
# Date 1689169165 -10800
#      Wed Jul 12 16:39:25 2023 +0300
# Node ID 7226cadb2d145a1feb5313c50eaadd87c6e49f51
# Parent  57f9cc8ef8ce24385ebb7bb4cf3150299bc8ed91
Core: Add ngx_strtok_r()

Add wrapper define for strtok_r().

diff -r 57f9cc8ef8ce -r 7226cadb2d14 src/core/ngx_string.h
--- a/src/core/ngx_string.h	Wed Jul 12 16:39:11 2023 +0300
+++ b/src/core/ngx_string.h	Wed Jul 12 16:39:25 2023 +0300
@@ -63,6 +63,8 @@
 size_t ngx_strnlen(u_char *p, size_t n);
 
 #define ngx_strchr(s1, c)   strchr((const char *) s1, (int) c)
+#define ngx_strtok_r(str, delim, saveptr) \
+    strtok_r((char *) str, (const char *) delim, (char **) saveptr)
 
 static ngx_inline u_char *
 ngx_strlchr(u_char *p, u_char *last, u_char c)

From vesa.jaaskelainen at vaisala.com  Wed Jul 12 14:07:06 2023
From: vesa.jaaskelainen at vaisala.com (=?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=)
Date: Wed, 12 Jul 2023 17:07:06 +0300
Subject: [PATCH 3/4] SSL: Add support for loading X.509 certificate chain from
 openssl engine
In-Reply-To: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
References: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
Message-ID: <20230712140707.364629-4-vesa.jaaskelainen@vaisala.com>

# HG changeset patch
# User Vesa Jääskeläinen <vesa.jaaskelainen at vaisala.com>
# Date 1689169177 -10800
#      Wed Jul 12 16:39:37 2023 +0300
# Node ID 30f709bbbd5a0754f1926a1a347348b68a1c487e
# Parent  7226cadb2d145a1feb5313c50eaadd87c6e49f51
SSL: Add support for loading X.509 certificate chain from openssl engine

Certificates within engine is often specified by URI, in order to support
specifying certificate chain use space as delimeter.

First certificate must match with ssl_certificate_key's key pair rest of
the certificiates are added to the certificate chain.

Code supports loading from different engines but normal case is expected
to be from same engine.

Example configuration with libp11's pkcs11 engine:

  ssl_certificate      "engine:pkcs11:pkcs11:token=mytoken;object=mykey
                        engine:pkcs11:pkcs11:token=mytoken;object=int-ca";
  ssl_certificate_key  "engine:pkcs11:pkcs11:token=mytoken;object=mykey?pin-value=mypin";

diff -r 7226cadb2d14 -r 30f709bbbd5a src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c	Wed Jul 12 16:39:25 2023 +0300
+++ b/src/event/ngx_event_openssl.c	Wed Jul 12 16:39:37 2023 +0300
@@ -638,14 +638,25 @@
 
 #ifndef OPENSSL_NO_ENGINE
 
-        u_char  *p, *last;
+        u_char  *p, *last, *cert_config, *saveptr;
         ENGINE  *engine;
 
-        p = cert->data + sizeof("engine:") - 1;
+        /*
+         * Certificates within engine is often specified by URI, in order to
+         * support specifying certificate chain use space as delimeter.
+         */
+
+        cert_config = ngx_pstrdup(pool, cert);
+
+        /* just add terminator at next delimeter */
+        (void) ngx_strtok_r(cert_config, " \t\r\n", &saveptr);
+
+        p = cert_config + sizeof("engine:") - 1;
         last = (u_char *) ngx_strchr(p, ':');
 
         if (last == NULL) {
             *err = "invalid syntax";
+            ngx_pfree(pool, cert_config);
             return NULL;
         }
 
@@ -655,6 +666,7 @@
 
         if (engine == NULL) {
             *err = "ENGINE_by_id() failed";
+            ngx_pfree(pool, cert_config);
             return NULL;
         }
 
@@ -664,12 +676,91 @@
 
         if (x509 == NULL) {
             *err = "ngx_ssl_engine_load_certificate() failed";
+            ngx_pfree(pool, cert_config);
             return NULL;
         }
 
         ENGINE_free(engine);
+        engine = NULL;
+
+        *chain = sk_X509_new_null();
+        if (*chain == NULL) {
+            *err = "sk_X509_new_null() failed";
+            X509_free(x509);
+            ngx_pfree(pool, cert_config);
+            return NULL;
+        }
+
+        /* load certificate chain if defined */
+        for ( ;; ) {
+            ngx_str_t chain_cert = ngx_null_string;
+
+            p = (u_char *) ngx_strtok_r(NULL, " \t\r\n", &saveptr);
+            if (!p) {
+                break;
+            }
+
+            ngx_str_set(&chain_cert, p);
+
+            if (ngx_strncmp(chain_cert.data, "engine:", sizeof("engine:") - 1)
+                == 0) {
+                p = chain_cert.data + sizeof("engine:") - 1;
+                last = (u_char *) ngx_strchr(p, ':');
+
+                if (last == NULL) {
+                    *err = "invalid syntax";
+                    X509_free(x509);
+                    sk_X509_pop_free(*chain, X509_free);
+                    ngx_pfree(pool, cert_config);
+                    return NULL;
+                }
+
+                *last = '\0';
+
+                engine = ENGINE_by_id((char *) p);
+
+                if (engine == NULL) {
+                    *err = "ENGINE_by_id() failed";
+                    X509_free(x509);
+                    sk_X509_pop_free(*chain, X509_free);
+                    ngx_pfree(pool, cert_config);
+                    return NULL;
+                }
+
+                *last++ = ':';
+
+                temp = ngx_ssl_engine_load_certificate(engine, (char *) last);
+
+                if (temp == NULL) {
+                    *err = "ngx_ssl_engine_load_certificate() failed for chain certificate";
+                    X509_free(x509);
+                    sk_X509_pop_free(*chain, X509_free);
+                    ENGINE_free(engine);
+                    ngx_pfree(pool, cert_config);
+                    return NULL;
+                }
+
+                if (sk_X509_push(*chain, temp) == 0) {
+                    *err = "sk_X509_push() failed";
+                    X509_free(x509);
+                    sk_X509_pop_free(*chain, X509_free);
+                    ENGINE_free(engine);
+                    ngx_pfree(pool, cert_config);
+                    return NULL;
+                }
+
+                ENGINE_free(engine);
+                engine = NULL;
+            } else {
+                *err = "loading \"engine:...\" certificate from different sources is not supported";
+                X509_free(x509);
+                sk_X509_pop_free(*chain, X509_free);
+                return NULL;
+            }
+        }
+
+        ngx_pfree(pool, cert_config);
         return x509;
-
 #else
 
         *err = "loading \"engine:...\" certificate is not supported";

From vesa.jaaskelainen at vaisala.com  Wed Jul 12 14:07:07 2023
From: vesa.jaaskelainen at vaisala.com (=?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=)
Date: Wed, 12 Jul 2023 17:07:07 +0300
Subject: [PATCH 4/4] SSL: Improve X.509 certificate loading from openssl
 engine with file source
In-Reply-To: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
References: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
Message-ID: <20230712140707.364629-5-vesa.jaaskelainen@vaisala.com>

# HG changeset patch
# User Vesa Jääskeläinen <vesa.jaaskelainen at vaisala.com>
# Date 1689169189 -10800
#      Wed Jul 12 16:39:49 2023 +0300
# Node ID 86d27e14c696e6c444b59063abb1e10fc31dfae6
# Parent  30f709bbbd5a0754f1926a1a347348b68a1c487e
SSL: Improve X.509 certificate loading from openssl engine with file source

In some cases key pair and its certificate are stored in engine but
intermediate certificates are not.

Add support for loading from the file system the rest of the chain.

As a side effect of using space as separator and when loading additional
certificates for chain parts from file system it cannot handle spaces in
the path.

Example configuration with libp11's pkcs11 engine:

  ssl_certificate      "engine:pkcs11:pkcs11:token=mytoken;object=mykey
                        /etc/ssl/certs/intermediate-ca.pem";
  ssl_certificate_key  "engine:pkcs11:pkcs11:token=mytoken;object=mykey?pin-value=mypin";

diff -r 30f709bbbd5a -r 86d27e14c696 src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c	Wed Jul 12 16:39:37 2023 +0300
+++ b/src/event/ngx_event_openssl.c	Wed Jul 12 16:39:49 2023 +0300
@@ -626,13 +626,48 @@
     return load_cert.cert;
 }
 
+static ngx_int_t
+ngx_ssl_bio_load_certificate_chain(BIO *bio, STACK_OF(X509) **chain, char **err)
+{
+    X509    *temp;
+    u_long   n;
+
+    for ( ;; ) {
+
+        temp = PEM_read_bio_X509(bio, NULL, NULL, NULL);
+        if (temp == NULL) {
+            n = ERR_peek_last_error();
+
+            if (ERR_GET_LIB(n) == ERR_LIB_PEM
+                && ERR_GET_REASON(n) == PEM_R_NO_START_LINE)
+            {
+                /* end of file */
+                ERR_clear_error();
+                break;
+            }
+
+            /* some real error */
+
+            *err = "PEM_read_bio_X509() failed";
+            return NGX_ERROR;
+        }
+
+        if (sk_X509_push(*chain, temp) == 0) {
+            *err = "sk_X509_push() failed";
+            return NGX_ERROR;
+        }
+    }
+
+    return NGX_OK;
+}
+
 static X509 *
 ngx_ssl_load_certificate(ngx_pool_t *pool, char **err, ngx_str_t *cert,
     STACK_OF(X509) **chain)
 {
-    BIO     *bio;
-    X509    *x509, *temp;
-    u_long   n;
+    BIO      *bio;
+    X509     *x509, *temp;
+    ngx_int_t ret;
 
     if (ngx_strncmp(cert->data, "engine:", sizeof("engine:") - 1) == 0) {
 
@@ -644,6 +679,9 @@
         /*
          * Certificates within engine is often specified by URI, in order to
          * support specifying certificate chain use space as delimeter.
+         *
+         * As a side effect loading additional certificates for chain from file
+         * system it cannot handle spaces in the path.
          */
 
         cert_config = ngx_pstrdup(pool, cert);
@@ -752,10 +790,49 @@
                 ENGINE_free(engine);
                 engine = NULL;
             } else {
-                *err = "loading \"engine:...\" certificate from different sources is not supported";
-                X509_free(x509);
-                sk_X509_pop_free(*chain, X509_free);
-                return NULL;
+                ngx_str_t cert_name = chain_cert;
+
+                if (ngx_get_full_name(pool,
+                                      (ngx_str_t *) &ngx_cycle->conf_prefix,
+                                      &cert_name)
+                    != NGX_OK)
+                {
+                    *err = "ngx_get_full_name() failed";
+                    X509_free(x509);
+                    sk_X509_pop_free(*chain, X509_free);
+                    ngx_pfree(pool, cert_config);
+                    return NULL;
+                }
+
+                bio = BIO_new_file((char *) cert_name.data, "r");
+                if (bio == NULL) {
+                    *err = "BIO_new_file() failed";
+                    if (cert_name.data != chain_cert.data) {
+                        /* new allocation -- free it */
+                        ngx_pfree(pool, cert_name.data);
+                    }
+                    X509_free(x509);
+                    sk_X509_pop_free(*chain, X509_free);
+                    ngx_pfree(pool, cert_config);
+                    return NULL;
+                }
+
+                if (cert_name.data != chain_cert.data) {
+                    /* new allocation -- free it */
+                    ngx_pfree(pool, cert_name.data);
+                }
+                ngx_str_null(&cert_name);
+
+                ret = ngx_ssl_bio_load_certificate_chain(bio, chain, err);
+                if (ret != NGX_OK) {
+                    BIO_free(bio);
+                    X509_free(x509);
+                    sk_X509_pop_free(*chain, X509_free);
+                    ngx_pfree(pool, cert_config);
+                    return NULL;
+                }
+
+                BIO_free(bio);
             }
         }
 
@@ -813,36 +890,12 @@
         return NULL;
     }
 
-    for ( ;; ) {
-
-        temp = PEM_read_bio_X509(bio, NULL, NULL, NULL);
-        if (temp == NULL) {
-            n = ERR_peek_last_error();
-
-            if (ERR_GET_LIB(n) == ERR_LIB_PEM
-                && ERR_GET_REASON(n) == PEM_R_NO_START_LINE)
-            {
-                /* end of file */
-                ERR_clear_error();
-                break;
-            }
-
-            /* some real error */
-
-            *err = "PEM_read_bio_X509() failed";
-            BIO_free(bio);
-            X509_free(x509);
-            sk_X509_pop_free(*chain, X509_free);
-            return NULL;
-        }
-
-        if (sk_X509_push(*chain, temp) == 0) {
-            *err = "sk_X509_push() failed";
-            BIO_free(bio);
-            X509_free(x509);
-            sk_X509_pop_free(*chain, X509_free);
-            return NULL;
-        }
+    ret = ngx_ssl_bio_load_certificate_chain(bio, chain, err);
+    if (ret != NGX_OK) {
+        BIO_free(bio);
+        X509_free(x509);
+        sk_X509_pop_free(*chain, X509_free);
+        return NULL;
     }
 
     BIO_free(bio);

From mdounin at mdounin.ru  Thu Jul 13 00:48:42 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 13 Jul 2023 03:48:42 +0300
Subject: [PATCH 0/4] SSL: Add support for loading X.509 certificates from
 openssl engine
In-Reply-To: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
References: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
Message-ID: <ZK9J6qdWPWYqSELG@mdounin.ru>

Hello!

On Wed, Jul 12, 2023 at 05:07:03PM +0300, Vesa Jääskeläinen via nginx-devel wrote:

> (I hope this goes properly out as I had major issues with hg email so
> combined hg export + git send-email)
> 
> It is convenient to keep X.509 certificates related to key pairs stored in
> openssl engine within the engine.
> 
> Implementation uses 'LOAD_CERT_CTRL' extension to fetch certificate from
> the engine. This extension is not supported by all engines and in those
> cases it should report with an error.
> 
> Configuration is similar to what it is for 'ssl_certificate_key'.
> 
> First certificate must match with ssl_certificate_key's key pair rest of
> the certificiates are added to the certificate chain.
> 
> Example configuration with libp11's pkcs11 engine:
> 
>   ssl_certificate      "engine:pkcs11:pkcs11:token=mytoken;object=mykey
>                         engine:pkcs11:pkcs11:token=mytoken;object=int-ca";
>   ssl_certificate_key  "engine:pkcs11:pkcs11:token=mytoken;object=mykey?pin-value=mypin";
> 
> Tested the loading with two pkcs11 implementations SoftHSMv2 and with
> OP-TEE's PKCS11 Trusted Application running on Embedded Linux device.
> 
> First three commits is the main beef and in order to make it more flexible
> added also last commit allowing intermediate certificates loaded from file
> system.
> 
> Separator of space is used as there was already existing use of array for
> ssl_certificate configuration.

Just in case, a similar proposal was previously discussed here:

https://mailman.nginx.org/pipermail/nginx-devel/2020-April/013130.html
https://mailman.nginx.org/pipermail/nginx-devel/2020-May/013142.html

Notably, the review is here:

https://mailman.nginx.org/pipermail/nginx-devel/2020-May/013152.html

I'm additionally sceptical about this given that engine interface 
is deprecated by OpenSSL.

-- 
Maxim Dounin
http://mdounin.ru/

From vesa.jaaskelainen at vaisala.com  Thu Jul 13 07:38:02 2023
From: vesa.jaaskelainen at vaisala.com (=?UTF-8?B?VmVzYSBKw6TDpHNrZWzDpGluZW4=?=)
Date: Thu, 13 Jul 2023 10:38:02 +0300
Subject: [PATCH 0/4] SSL: Add support for loading X.509 certificates from
 openssl engine
In-Reply-To: <ZK9J6qdWPWYqSELG@mdounin.ru>
References: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
 <ZK9J6qdWPWYqSELG@mdounin.ru>
Message-ID: <e6d3b076-b400-3be1-bf4a-a6d761978ff8@vaisala.com>

Hi Maxim,

On 13.7.2023 3.48, Maxim Dounin wrote:
> Hello!
>
> On Wed, Jul 12, 2023 at 05:07:03PM +0300, Vesa Jääskeläinen via nginx-devel wrote:
>
>> (I hope this goes properly out as I had major issues with hg email so
>> combined hg export + git send-email)
>>
>> It is convenient to keep X.509 certificates related to key pairs stored in
>> openssl engine within the engine.
>>
>> Implementation uses 'LOAD_CERT_CTRL' extension to fetch certificate from
>> the engine. This extension is not supported by all engines and in those
>> cases it should report with an error.
>>
>> Configuration is similar to what it is for 'ssl_certificate_key'.
>>
>> First certificate must match with ssl_certificate_key's key pair rest of
>> the certificiates are added to the certificate chain.
>>
>> Example configuration with libp11's pkcs11 engine:
>>
>>    ssl_certificate      "engine:pkcs11:pkcs11:token=mytoken;object=mykey
>>                          engine:pkcs11:pkcs11:token=mytoken;object=int-ca";
>>    ssl_certificate_key  "engine:pkcs11:pkcs11:token=mytoken;object=mykey?pin-value=mypin";
>>
>> Tested the loading with two pkcs11 implementations SoftHSMv2 and with
>> OP-TEE's PKCS11 Trusted Application running on Embedded Linux device.
>>
>> First three commits is the main beef and in order to make it more flexible
>> added also last commit allowing intermediate certificates loaded from file
>> system.
>>
>> Separator of space is used as there was already existing use of array for
>> ssl_certificate configuration.
> Just in case, a similar proposal was previously discussed here:
>
> https://mailman.nginx.org/pipermail/nginx-devel/2020-April/013130.html
> https://mailman.nginx.org/pipermail/nginx-devel/2020-May/013142.html
>
> Notably, the review is here:
>
> https://mailman.nginx.org/pipermail/nginx-devel/2020-May/013152.html
>
> I'm additionally sceptical about this given that engine interface
> is deprecated by OpenSSL.

Thanks for the links.

The provider interface in OpenSSL 3 is yet to be matured for PKCS11 usage.

libp11 so far seems to have stayed with "legacy" engine and just adapted 
for OpenSSL 3.

Then there is new project that is still in development phase but looks 
promising:

https://github.com/latchset/pkcs11-provider

But before that provider implementation matures and is available in 
distributions it takes one or more major LTS releases.

While waiting for that it would be nice to have the support.

What is interesting that the CMD extension is already being integrated 
to other software like curl, wpa-supplicant/hostapd, stunnel, ppp, 
M2Crypto -- so I would say the feature is there to stay for a while. 
Even not documented as official openssl "extension" it is there and is 
being used.

Now openssl also has "store" API which can be extended with different 
schemes too but in example libp11 does not today support that.

https://www.openssl.org/docs/man1.1.1/man7/ossl_store.html
https://www.openssl.org/docs/man1.1.1/man3/OSSL_STORE_LOADER.html

Now the Nginx is already using the "legacy" interface so if it is enable 
why not then support the certificate loading when it is available.

I believe these can even coexist.

   provider:<provider>:<object id>
   provider:pkcs11:<your pkcs11 uri>

and

   engine:<engine>:<object id>
   engine:pkcs11:<your pkcs11 uri>

or even:

   store:<scheme>:<object id>
   store:pkcs11:<your pkcs11 uri>
   store:file:/path/to/your/pem-file

Now the problem of these all is what certificate to load and what 
certificates from the chain to load if any.

"The correct way" is to load all certificates till the root (excluding 
the root) but this is not how some people setup the stuff. So you need a 
way to be able to specify the chain.

With PEM files this is easy as one can craft them freely.

With other providers you need to either configure (like what M2Crypto 
uses) or be simple only use the certificate without chain (a bit ugly 
approach).

   ssl_certificate "engine:pkcs11:<your pkcs11 uri> engine:pkcs11:<your 
pkcs11 uri> engine:pkcs11:<your pkcs11 uri>"

or you could even mix'n'match if you have go good idea:

   ssl_certificate "provider:<provider>:<object id> 
store:<scheme>:<object id> /my/pem/file/here.pem"

For this I used the white space(s) as a separator in order to be able to 
configure the chain.

Alternative for that would have been:

   ssl_certificate "engine:pkcs11:<your pkcs11 uri>" 
"engine:pkcs11:<your pkcs11 uri>" "engine:pkcs11:<your pkcs11 uri>"

But that would have created array of array of strings as there was 
support to configure multiple certificate chains+key pairs and let the 
nginx pick which to serve.

Why I believe it is a good idea to support engine/or in future the 
provider interface is that to keep the certificates in one place next to 
the keys. With this if the key pair gets re-created or new certificates 
to be updated if you have the labels there you just need to trigger the 
refresh. Then there is no extra data to extract or such as then those 
could be out-of-sync and people try to figure out problems with that.

If I am not mistaken the previous discussion comments has been addressed 
in this change set.

I am happy to test out the provider with pkcs11-provider too but I would 
recommend that we also support today's methods so people can start using 
the feature.

Thanks,
Vesa Jääskeläinen

From mdounin at mdounin.ru  Thu Jul 13 15:35:44 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 13 Jul 2023 18:35:44 +0300
Subject: Fixing the well Documented Nginx Alias Traversal Vulnerability?
In-Reply-To: <CAJq_=OU=0sWhvcwgPDAkRM8i7jhYFCx5GUS4Mt+QHWLDBZCGWQ@mail.gmail.com>
References: <CAJq_=OXAOdULC4jnngiLJefcJNR6_hFrL0+34wAuCr-7W7hLkg@mail.gmail.com>
 <ZKieDyPi5c8PXdiP@mdounin.ru>
 <CAJq_=OU=0sWhvcwgPDAkRM8i7jhYFCx5GUS4Mt+QHWLDBZCGWQ@mail.gmail.com>
Message-ID: <ZLAZ0FsBMrvGJEms@mdounin.ru>

Hello!

On Wed, Jul 12, 2023 at 05:45:34PM -0400, Jonathan Leitschuh wrote:

> Hi Maxim,
> 
> Pleasure to meet you!
> 
> I've also included Munawar, the CEO of Open Refactory in this email chain.
> 
> > (It looks like you aren't subscribed to nginx-devel@, so the message did
> not reach the mailing list: it only accepts messages from subscribers.
> I've preserved it in Cc: though, so we can continue discussion there, with
> more developers involved.)
> 
> Do I need to subscribe to continue to get this thread included in that
> email group? The website didn't explicitly say that I needed to subscribe
> to send emails there, that might be a good addition.

The http://nginx.org/en/support.html page says:

"To post to a mailing list, an e-mail address that will be used 
for posting must first be subscribed."

This applies to all mailing lists on the page, including 
nginx-devel at .  Sorry if it wasn't clear.

> > Indeed, the misconfiguration in question is a well-known one. Gixy, a
> tool to detect various nginx misconfigurations, tries to detect it at least
> since 2017:
> >
> >
> https://github.com/yandex/gixy/blob/master/docs/en/plugins/aliastraversal.md
> 
> From the documentation, they provide the following example request:
> `/i../app/config.py`. I can't imagine a case where a request like
> `/i../app/config.py` could ever be made legitimately by a user. Why is
> `../app/config.py` appended to the alias `/data/w3/images/` in this case
> without nginx throwing an exception as this is a blatant path traversal
> attack?

The "i.." string is a valid path component and there are no 
reasons why it shouldn't be allowed - there might be such files 
and/or directories on the disk.  Similarly, location matching 
and alias as implemented in nginx (and documented) are expected to 
result in "../app/config.py" appended to "/data/w3/images/" for 
such a configuration.

The question here is why the location prefix without trailing "/" 
was used in the configuration, and if it was intentional.

> > Unfortunately, I don't think there is an easy fix.  Changing the way how
> nginx locations work would be a major change, and will break a lot of valid
> configurations where prefix matching is intentionally used.
> 
> This sounds like a fix that could this go through a depreciation cycle,
> where, nginx adds an intentional, opt-in, prefix-matching flag that, in
> it's early incarnations, offers warnings to users for
> deprecated configuration, and later, will fail-closed unless the flag is
> present?

Sure, major changes can be implemented in a way which warns users 
in advance.  This still be a major change though, and the 
transition will require several years and will disturb a lot of 
users, including ones who aren't affected by the configuration 
issue in question.

Further, it is not clear if it will help: nothing will stop users 
from using the explicit flag for prefix matching in an unsafe way 
without realizing it's unsafe.

Given that, an explicit warning/note in the documentation looks 
like the way to go, at least for now.

> 
> > If you think there is an easy fix, feel free to provide suggestions.
> 
> On our side, between Munwar and myself, we have some significant experience
> with bulk pull request generation to fix security vulnerabilities at-scale
> across open source software. (See my Black Hat & DEF CON Talk on the topic
> from last year
> <https://github.com/jlleitschuh/#scaling-the-security-researcher-to-eliminate-oss-vulnerabilities-once-and-for-all>
> )
> 
> Munawar and his team at OpenRefactory did an analysis of the 2k+
> repositories that GitHub flags with the above linked search. From his
> team's analysis:
> 
>  - Majority of the bugs are in documentation. These can be easily filtered
> out with the file type. We can consider bulk pull request generation adding
> a bit that we are fixing a documentation to make it correct so that people
> are not misinformed and left with a vulnerability.
>  - There are a few bugs in the actual config files. In such cases, you may
> split the projects based on the GH stars. The more popular ones should get
> a stronger message communicating the risk to their end-users.
> 
> We can likely massively clean-up this vulnerability across the OSS
> ecosystem, but the long-term solution to this vulnerability, I believe,
> will lie in the hands of the Nginx team.
> 
> How can we work together here towards a world where this vulnerability is
> something end-users opt-in to, with intention (usually via some flag),
> rather than the world we are currently in, where this vulnerability appears
> as a mistake?
> 
> Cheers,
> Jonathan Leitschuh
> 
> 
> On Fri, Jul 7, 2023 at 7:21 PM Maxim Dounin <mdounin at mdounin.ru> wrote:
> 
> > Hello!
> >
> > On Thu, Jul 06, 2023 at 12:05:25PM -0400, Jonathan Leitschuh wrote:
> >
> > > Hi Nginx Team,
> > >
> > > My name is Jonathan Leitschuh. I'm a Senior Software Security Researcher
> > > for the Open Source Security Foundation: Project Alpha-Omega
> > > <https://openssf.org/community/alpha-omega/>. “Alpha” works with the
> > > maintainers of the most critical open source projects to help them
> > identify
> > > and fix security vulnerabilities, and improve their security posture.
> > > “Omega” identified at least 10,000 widely deployed OSS projects where it
> > > can apply automated security analysis, scoring, and remediation guidance
> > to
> > > their open source maintainer communities.
> > >
> > > Earlier today, on Twitter, I saw this discussion regarding
> > > the long-standing the NGinx Alias traversal vulnerability:
> > > https://twitter.com/infosec_au/status/1676072447156834304?s=20
> > >
> > > In particular, this discussion centered on this, this recent blog post
> > > published on the 3rd of this month.
> > > https://labs.hakaioffsec.com/nginx-alias-traversal/
> > >
> > > In that blog post, the researchers detail how, due to misconfiguration
> > and
> > > failing to specify the `/` character in the correct place can lead to
> > path
> > > traversal vulnerabilities.
> > >
> > >
> > >
> > > This vulnerability was previously discussed by Detectify in a blog post
> > in
> > > 2020:
> > > https://blog.detectify.com/2020/11/10/common-nginx-misconfigurations/
> > >
> > > This research originated with Orange Tsai back in 2018:
> > >
> > https://i.blackhat.com/us-18/Wed-August-8/us-18-Orange-Tsai-Breaking-Parser-Logic-Take-Your-Path-Normalization-Off-And-Pop-0days-Out-2.pdf
> > >
> > > The recent hakaioffsec blog also identifies this case where the
> > > vulnerability also appears:
> > >
> > >
> > > My question to both the Nginx Security team, and the Nginx Developers,
> > why
> > > not fix this vulnerability in Nginx itself such that it's safe by
> > default,
> > > but users can opt-in to the vulnerable behaviour if they so desire
> > through
> > > an explicit flag? Surely it would be fairly straightforward to identify
> > > these vulnerable configurations and fix them for the user, or fail closed
> > > (safe)?
> > >
> > > If you need evidence of the impact this vulnerability can have, the
> > > above hakaioffsec blog post details how this vulnerability resulted in
> > > a US$6000 bounty from BitWarden.
> > >
> > > This GitHub search, proposed by the above blog post, returns 2.2k results
> > > on GitHub:
> > >
> > >
> > https://github.com/search?q=%2Flocation+%5C%2F%5B_.a-zA-Z0-9-%5C%2F%5D*%5B%5E%5C%2F%5D%5B%5Cs%5D%5C%7B%5B%5Cs%5Cn%5D*alias+%5C%2F%5B_.a-zA-Z0-9-%5C%2F%5D*%5C%2F%3B%2F+&type=code
> > >
> > > I'm curious what is currently preventing this from being fixed upstream
> > to
> > > prevent end-users from accidentally implementing vulnerable
> > configurations
> > > like this.
> >
> > Thank you for your message.
> >
> > Indeed, the misconfiguration in question is a well-known one.
> > Gixy, a tool to detect various nginx misconfigurations, tries to
> > detect it at least since 2017:
> >
> >
> > https://github.com/yandex/gixy/blob/master/docs/en/plugins/aliastraversal.md
> >
> > The root cause as I see it is that nginx locations (and alias)
> > work with prefix strings, while unaware people often expect them
> > to work with path components (directories) instead.  Note the
> > "Achieving Impact Without a Slash on Alias" section in the article
> > you've referenced: a configuration like
> >
> >     location /img {
> >         alias /var/images;
> >     }
> >
> > makes it possible to access "/var/images_confidential" via a
> > request to "/img_confidential".  This is because locations works
> > on prefix strings, and due to lack of trailing "/" in the location
> > prefix it does match "/img_confidential" request, and maps it to
> > the file system according to the configuration.
> >
> > Similarly, the same misconfiguration can happen even without the
> > "alias" directive at all. Consider the configuration:
> >
> >     location / { deny all; }
> >     location /images { allow all; }
> >
> > In contrast to some might (incorrectly) assume, a request to
> > "/images_confidential" will be allowed, and will return content as
> > per the configuration.
> >
> > Unfortunately, I don't think there is an easy fix.  Changing the
> > way how nginx locations work would be a major change, and will
> > break a lot of valid configurations where prefix matching is
> > intentionally used.
> >
> > Meanwhile, it surely deserves some additional warnings/notes in
> > the documentation, to emphasize the actual behaviour and potential
> > pitfalls.  Hopefully it would be enough to at least reduce the
> > amount of such misconfigurations.
> >
> > If you think there is an easy fix, feel free to provide
> > suggestions.
> >
> > (It looks like you aren't subscribed to nginx-devel@, so the
> > message did not reach the mailing list: it only accepts messages
> > from subscribers.  I've preserved it in Cc: though, so we can
> > continue discussion there, with more developers involved.)
> >
> > --
> > Maxim Dounin
> > http://mdounin.ru/
> >

-- 
Maxim Dounin
http://mdounin.ru/

From pluknet at nginx.com  Thu Jul 13 16:58:44 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Thu, 13 Jul 2023 20:58:44 +0400
Subject: [PATCH] Core: fixed environment on exit better
In-Reply-To: <ZJZPtrxW0uDtw/57@mdounin.ru>
References: <34a8e1a4161542896c11.1687535747@enoparse.local>
 <ZJZPtrxW0uDtw/57@mdounin.ru>
Message-ID: <74BC1CA2-9522-43AA-9EFD-24581BF6F3FB@nginx.com>


> On 24 Jun 2023, at 06:06, Maxim Dounin <mdounin at mdounin.ru> wrote:
> 
> Hello!
> 
> On Fri, Jun 23, 2023 at 07:55:47PM +0400, Sergey Kandaurov wrote:
> 
>> # HG changeset patch
>> # User Sergey Kandaurov <pluknet at nginx.com>
>> # Date 1687535739 -14400
>> #      Fri Jun 23 19:55:39 2023 +0400
>> # Node ID 34a8e1a4161542896c11c4a5c60d6a6fe1931e3d
>> # Parent  c681f4906d838ac45b517fada4c4274ade03fd3c
>> Core: fixed environment on exit better.
>> 
>> The fix in 6822:c045b4926b2c to replace environment allocations from a pool
>> was incomplete, such that it left pool owned allocations for variables set
>> with values using the "env" directive.  Another consumer of this interface
>> is the QUIC BPF helper that uses the environment variable NGINX_BPF_MAPS.
>> The fix is to re-allocate variables with malloc, to avoid use after free.
>> If memory allocation failed, the only option is to expel from environment.
>> 
>> The observed symptoms are similar to described in 6822:c045b4926b2c, that
>> is a segmentation fault on worker process exit from atexit() handler seen
>> with 3rd party modules or if nginx was built with lcov profiling.
>> 
>> diff --git a/src/core/nginx.c b/src/core/nginx.c
>> --- a/src/core/nginx.c
>> +++ b/src/core/nginx.c
>> @@ -631,17 +631,48 @@ ngx_cleanup_environment(void *data)
>> {
>>     char  **env = data;
>> 
>> -    if (environ == env) {
>> +    char        *p, **penv;
>> +    size_t       size;
>> +    ngx_uint_t   i, n;
>> 
>> -        /*
>> -         * if the environment is still used, as it happens on exit,
>> -         * the only option is to leak it
>> -         */
>> -
>> +    if (environ != env) {
>> +        ngx_free(env);
>>         return;
>>     }
> 
> Note: if we assume that the environment can be arbitrary modified, 
> I don't think it's enough.  For example, a setenv() call can 
> replace environ with something different, so this code will free 
> env and exit, but pool-allocated variables will be still 
> referenced by the new environ.
> 
> This is probably out of the scope of this patch though.

It could be solved by always checking for pool-allocated variables,
irrespective of whether environ was changed after setenv(),
using that as an additional condition to free environment.

> 
>> 
>> -    ngx_free(env);
>> +    /*
>> +     * if the environment is still used, as it happens on exit,
>> +     * the only option is to leak it
>> +     */
>> +
>> +    for (n = 0; env[n]; n++) { }
>> +
>> +    for (i = 0; env[i]; i++) {
>> +
>> +        for (penv = ngx_os_environ; *penv; penv++) {
>> +            if (ngx_strcmp(*penv, env[i]) == 0) {
> 
> I don't think it's going to work.  If I'm reading this correctly, 
> with the initial environment "FOO=bar" and "env FOO=bar;" in the 
> configuration this will assume the string is from ngx_os_environ, 
> but it will be from ccf->env instead.
> 

Thanks for catching that.
So, we have to compare against ccf->env, instead.

>> +                break;
>> +            }
>> +        }
>> +
>> +        if (*penv) {
>> +            continue;
>> +        }
>> +
>> +        /* substitute pool allocated variables */
>> +
>> +        size = ngx_strlen(env[i]) + 1;
>> +        p = malloc(size);
> 
> There should be no direct malloc() calls except in low-level code 
> where logging is not available.  I don't think this particular 
> place counts, so ngx_alloc() should be used instead.

I used to avoid using cycle in cleanup, thus direct malloc() calls.
Anyway, it will be needed in order to access configuration.

Below is an updated patch (commit log unchanged).

# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1689267366 -14400
#      Thu Jul 13 20:56:06 2023 +0400
# Node ID 80a60d9707ed4b80402335d3d31a978f418f8056
# Parent  f91dc350be9f4a6bf1379a32a210afece7b0d75e
Core: fixed environment on exit better.

The fix in 6822:c045b4926b2c to replace environment allocations from a pool
was incomplete, such that it left pool owned allocations for variables set
with values using the "env" directive.  Another consumer of this interface
is the QUIC BPF helper that uses the environment variable NGINX_BPF_MAPS.
The fix is to re-allocate variables with malloc, to avoid use after free.
If memory allocation failed, the only option is to expel from environment.

The observed symptoms are similar to described in 6822:c045b4926b2c, that
is a segmentation fault on worker process exit from atexit() handler seen
with 3rd party modules or if nginx was built with lcov profiling.

diff --git a/src/core/nginx.c b/src/core/nginx.c
--- a/src/core/nginx.c
+++ b/src/core/nginx.c
@@ -592,7 +592,7 @@ tz_found:
         }
 
         cln->handler = ngx_cleanup_environment;
-        cln->data = env;
+        cln->data = cycle;
     }
 
     n = 0;
@@ -629,19 +629,57 @@ tz_found:
 static void
 ngx_cleanup_environment(void *data)
 {
-    char  **env = data;
-
-    if (environ == env) {
+    ngx_cycle_t  *cycle = data;
 
-        /*
-         * if the environment is still used, as it happens on exit,
-         * the only option is to leak it
-         */
+    char             *p, **env;
+    size_t            size;
+    ngx_str_t        *var;
+    ngx_uint_t        i, n;
+    ngx_core_conf_t  *ccf;
 
+    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
+    env = ccf->environment;
+
+    if (environ != env) {
+        ngx_free(env);
         return;
     }
 
-    ngx_free(env);
+    /*
+     * if the environment is still used, as it happens on exit,
+     * the only option is to leak it
+     */
+
+    for (n = 0; env[n]; n++) { /* void */ }
+
+    while (*env) {
+        var = ccf->env.elts;
+
+        for (i = 0; i < ccf->env.nelts; i++) {
+            if (var[i].data == (u_char *) *env) {
+                break;
+            }
+        }
+
+        if (i == ccf->env.nelts) {
+            env++;
+            continue;
+        }
+
+        /* substitute pool allocated variables */
+
+        size = ngx_strlen(*env) + 1;
+        p = ngx_alloc(size, cycle->log);
+
+        if (p) {
+            ngx_memcpy(p, *env, size);
+            *env++ = p;
+
+        } else {
+            ngx_memmove(env, env + 1, sizeof(char *) * (n - (env - environ)));
+            n--;
+        }
+    }
 }
 
 

> 
>> +
>> +        if (p) {
>> +            ngx_memcpy(p, env[i], size);
>> +            env[i] = p;
>> +
>> +        } else {
>> +            ngx_memmove(&env[i], &env[i + 1], sizeof(char *) * (n - i));
>> +            i--; n--;
>> +        }
>> +    }
>> }
> 
> Overall, it looks like a better idea might be to simply add all 
> the variables added to the environment to the environ allocation 
> in ngx_set_environment().
> 
> In ngx_set_environment() we explicitly know if the particular 
> variable is from OS environment or from ccf->environ, so there 
> will be no need to guess.  Also, the appropriate log is readily 
> available (and already used in the existing ngx_alloc() call), and 
> the code to cleanup things is already there and will work without 
> additional modifications.

I tried to avoid allocations in ngx_set_environment(),
as it can be called from the master process and harm
hypothetically by accumulating leaked allocations.

-- 
Sergey Kandaurov

From Jonathan.Leitschuh at linuxfoundation.org  Fri Jul 14 22:50:54 2023
From: Jonathan.Leitschuh at linuxfoundation.org (Jonathan Leitschuh)
Date: Fri, 14 Jul 2023 18:50:54 -0400
Subject: Fixing the well Documented Nginx Alias Traversal Vulnerability?
In-Reply-To: <ZLAZ0FsBMrvGJEms@mdounin.ru>
References: <CAJq_=OXAOdULC4jnngiLJefcJNR6_hFrL0+34wAuCr-7W7hLkg@mail.gmail.com>
 <ZKieDyPi5c8PXdiP@mdounin.ru>
 <CAJq_=OU=0sWhvcwgPDAkRM8i7jhYFCx5GUS4Mt+QHWLDBZCGWQ@mail.gmail.com>
 <ZLAZ0FsBMrvGJEms@mdounin.ru>
Message-ID: <CAJq_=OVy9ZmhHfBWvbdA0pG6_QGB1jg_-LBHQsy_VcP+-Bt2cQ@mail.gmail.com>

Hi Maxim,

> The "i.." string is a valid path component and there are no reasons why
it shouldn't be allowed - there might be such files and/or directories on
the disk.

Do you have any metrics on how often this is truly used? Or how common this
file path really is? Is the argument here that "this might be valid, so we
should support it" or is there some wider use case of this standard that
I'm unfamiliar with? It seems puzzling to me to support a potential file
name like "i.." if it's frequency of use is incredibly low, and, more often
than not, likely to result in a security vulnerability.

> Similarly, location matching and alias as implemented in nginx (and
documented) are expected to result in "../app/config.py" appended to
"/data/w3/images/" for such a configuration.

Speaking as a security researcher, this seems blatantly like behaviour that
most people would never want intentionally. The only valid use cases I
could see would be as an intentional vulnerability introduced by someone
building an intentionally vulnerable application for something like a
security capture-the-flag. But in those cases, the vulnerability is
intentionally left in place, instead of accidentally introduced. I don't,
personally, have any concerns with breaking those cases. Are you aware of
any valid uses cases of this behaviour in the real world, or is this
behaviour left because of hypothetical configurations that this may break?

Cheers,
Jonathan Leitschuh


On Thu, Jul 13, 2023 at 11:35 AM Maxim Dounin <mdounin at mdounin.ru> wrote:

> Hello!
>
> On Wed, Jul 12, 2023 at 05:45:34PM -0400, Jonathan Leitschuh wrote:
>
> > Hi Maxim,
> >
> > Pleasure to meet you!
> >
> > I've also included Munawar, the CEO of Open Refactory in this email
> chain.
> >
> > > (It looks like you aren't subscribed to nginx-devel@, so the message
> did
> > not reach the mailing list: it only accepts messages from subscribers.
> > I've preserved it in Cc: though, so we can continue discussion there,
> with
> > more developers involved.)
> >
> > Do I need to subscribe to continue to get this thread included in that
> > email group? The website didn't explicitly say that I needed to subscribe
> > to send emails there, that might be a good addition.
>
> The http://nginx.org/en/support.html page says:
>
> "To post to a mailing list, an e-mail address that will be used
> for posting must first be subscribed."
>
> This applies to all mailing lists on the page, including
> nginx-devel at .  Sorry if it wasn't clear.
>
> > > Indeed, the misconfiguration in question is a well-known one. Gixy, a
> > tool to detect various nginx misconfigurations, tries to detect it at
> least
> > since 2017:
> > >
> > >
> >
> https://github.com/yandex/gixy/blob/master/docs/en/plugins/aliastraversal.md
> >
> > From the documentation, they provide the following example request:
> > `/i../app/config.py`. I can't imagine a case where a request like
> > `/i../app/config.py` could ever be made legitimately by a user. Why is
> > `../app/config.py` appended to the alias `/data/w3/images/` in this case
> > without nginx throwing an exception as this is a blatant path traversal
> > attack?
>
> The "i.." string is a valid path component and there are no
> reasons why it shouldn't be allowed - there might be such files
> and/or directories on the disk.  Similarly, location matching
> and alias as implemented in nginx (and documented) are expected to
> result in "../app/config.py" appended to "/data/w3/images/" for
> such a configuration.
>
> The question here is why the location prefix without trailing "/"
> was used in the configuration, and if it was intentional.
>
> > > Unfortunately, I don't think there is an easy fix.  Changing the way
> how
> > nginx locations work would be a major change, and will break a lot of
> valid
> > configurations where prefix matching is intentionally used.
> >
> > This sounds like a fix that could this go through a depreciation cycle,
> > where, nginx adds an intentional, opt-in, prefix-matching flag that, in
> > it's early incarnations, offers warnings to users for
> > deprecated configuration, and later, will fail-closed unless the flag is
> > present?
>
> Sure, major changes can be implemented in a way which warns users
> in advance.  This still be a major change though, and the
> transition will require several years and will disturb a lot of
> users, including ones who aren't affected by the configuration
> issue in question.
>
> Further, it is not clear if it will help: nothing will stop users
> from using the explicit flag for prefix matching in an unsafe way
> without realizing it's unsafe.
>
> Given that, an explicit warning/note in the documentation looks
> like the way to go, at least for now.
>
> >
> > > If you think there is an easy fix, feel free to provide suggestions.
> >
> > On our side, between Munwar and myself, we have some significant
> experience
> > with bulk pull request generation to fix security vulnerabilities
> at-scale
> > across open source software. (See my Black Hat & DEF CON Talk on the
> topic
> > from last year
> > <
> https://github.com/jlleitschuh/#scaling-the-security-researcher-to-eliminate-oss-vulnerabilities-once-and-for-all
> >
> > )
> >
> > Munawar and his team at OpenRefactory did an analysis of the 2k+
> > repositories that GitHub flags with the above linked search. From his
> > team's analysis:
> >
> >  - Majority of the bugs are in documentation. These can be easily
> filtered
> > out with the file type. We can consider bulk pull request generation
> adding
> > a bit that we are fixing a documentation to make it correct so that
> people
> > are not misinformed and left with a vulnerability.
> >  - There are a few bugs in the actual config files. In such cases, you
> may
> > split the projects based on the GH stars. The more popular ones should
> get
> > a stronger message communicating the risk to their end-users.
> >
> > We can likely massively clean-up this vulnerability across the OSS
> > ecosystem, but the long-term solution to this vulnerability, I believe,
> > will lie in the hands of the Nginx team.
> >
> > How can we work together here towards a world where this vulnerability is
> > something end-users opt-in to, with intention (usually via some flag),
> > rather than the world we are currently in, where this vulnerability
> appears
> > as a mistake?
> >
> > Cheers,
> > Jonathan Leitschuh
> >
> >
> > On Fri, Jul 7, 2023 at 7:21 PM Maxim Dounin <mdounin at mdounin.ru> wrote:
> >
> > > Hello!
> > >
> > > On Thu, Jul 06, 2023 at 12:05:25PM -0400, Jonathan Leitschuh wrote:
> > >
> > > > Hi Nginx Team,
> > > >
> > > > My name is Jonathan Leitschuh. I'm a Senior Software Security
> Researcher
> > > > for the Open Source Security Foundation: Project Alpha-Omega
> > > > <https://openssf.org/community/alpha-omega/>. “Alpha” works with the
> > > > maintainers of the most critical open source projects to help them
> > > identify
> > > > and fix security vulnerabilities, and improve their security posture.
> > > > “Omega” identified at least 10,000 widely deployed OSS projects
> where it
> > > > can apply automated security analysis, scoring, and remediation
> guidance
> > > to
> > > > their open source maintainer communities.
> > > >
> > > > Earlier today, on Twitter, I saw this discussion regarding
> > > > the long-standing the NGinx Alias traversal vulnerability:
> > > > https://twitter.com/infosec_au/status/1676072447156834304?s=20
> > > >
> > > > In particular, this discussion centered on this, this recent blog
> post
> > > > published on the 3rd of this month.
> > > > https://labs.hakaioffsec.com/nginx-alias-traversal/
> > > >
> > > > In that blog post, the researchers detail how, due to
> misconfiguration
> > > and
> > > > failing to specify the `/` character in the correct place can lead to
> > > path
> > > > traversal vulnerabilities.
> > > >
> > > >
> > > >
> > > > This vulnerability was previously discussed by Detectify in a blog
> post
> > > in
> > > > 2020:
> > > >
> https://blog.detectify.com/2020/11/10/common-nginx-misconfigurations/
> > > >
> > > > This research originated with Orange Tsai back in 2018:
> > > >
> > >
> https://i.blackhat.com/us-18/Wed-August-8/us-18-Orange-Tsai-Breaking-Parser-Logic-Take-Your-Path-Normalization-Off-And-Pop-0days-Out-2.pdf
> > > >
> > > > The recent hakaioffsec blog also identifies this case where the
> > > > vulnerability also appears:
> > > >
> > > >
> > > > My question to both the Nginx Security team, and the Nginx
> Developers,
> > > why
> > > > not fix this vulnerability in Nginx itself such that it's safe by
> > > default,
> > > > but users can opt-in to the vulnerable behaviour if they so desire
> > > through
> > > > an explicit flag? Surely it would be fairly straightforward to
> identify
> > > > these vulnerable configurations and fix them for the user, or fail
> closed
> > > > (safe)?
> > > >
> > > > If you need evidence of the impact this vulnerability can have, the
> > > > above hakaioffsec blog post details how this vulnerability resulted
> in
> > > > a US$6000 bounty from BitWarden.
> > > >
> > > > This GitHub search, proposed by the above blog post, returns 2.2k
> results
> > > > on GitHub:
> > > >
> > > >
> > >
> https://github.com/search?q=%2Flocation+%5C%2F%5B_.a-zA-Z0-9-%5C%2F%5D*%5B%5E%5C%2F%5D%5B%5Cs%5D%5C%7B%5B%5Cs%5Cn%5D*alias+%5C%2F%5B_.a-zA-Z0-9-%5C%2F%5D*%5C%2F%3B%2F+&type=code
> > > >
> > > > I'm curious what is currently preventing this from being fixed
> upstream
> > > to
> > > > prevent end-users from accidentally implementing vulnerable
> > > configurations
> > > > like this.
> > >
> > > Thank you for your message.
> > >
> > > Indeed, the misconfiguration in question is a well-known one.
> > > Gixy, a tool to detect various nginx misconfigurations, tries to
> > > detect it at least since 2017:
> > >
> > >
> > >
> https://github.com/yandex/gixy/blob/master/docs/en/plugins/aliastraversal.md
> > >
> > > The root cause as I see it is that nginx locations (and alias)
> > > work with prefix strings, while unaware people often expect them
> > > to work with path components (directories) instead.  Note the
> > > "Achieving Impact Without a Slash on Alias" section in the article
> > > you've referenced: a configuration like
> > >
> > >     location /img {
> > >         alias /var/images;
> > >     }
> > >
> > > makes it possible to access "/var/images_confidential" via a
> > > request to "/img_confidential".  This is because locations works
> > > on prefix strings, and due to lack of trailing "/" in the location
> > > prefix it does match "/img_confidential" request, and maps it to
> > > the file system according to the configuration.
> > >
> > > Similarly, the same misconfiguration can happen even without the
> > > "alias" directive at all. Consider the configuration:
> > >
> > >     location / { deny all; }
> > >     location /images { allow all; }
> > >
> > > In contrast to some might (incorrectly) assume, a request to
> > > "/images_confidential" will be allowed, and will return content as
> > > per the configuration.
> > >
> > > Unfortunately, I don't think there is an easy fix.  Changing the
> > > way how nginx locations work would be a major change, and will
> > > break a lot of valid configurations where prefix matching is
> > > intentionally used.
> > >
> > > Meanwhile, it surely deserves some additional warnings/notes in
> > > the documentation, to emphasize the actual behaviour and potential
> > > pitfalls.  Hopefully it would be enough to at least reduce the
> > > amount of such misconfigurations.
> > >
> > > If you think there is an easy fix, feel free to provide
> > > suggestions.
> > >
> > > (It looks like you aren't subscribed to nginx-devel@, so the
> > > message did not reach the mailing list: it only accepts messages
> > > from subscribers.  I've preserved it in Cc: though, so we can
> > > continue discussion there, with more developers involved.)
> > >
> > > --
> > > Maxim Dounin
> > > http://mdounin.ru/
> > >
>
> --
> Maxim Dounin
> http://mdounin.ru/
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20230714/80654d2f/attachment-0001.htm>

From mdounin at mdounin.ru  Sat Jul 15 01:42:10 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Sat, 15 Jul 2023 04:42:10 +0300
Subject: [PATCH 0/4] SSL: Add support for loading X.509 certificates from
 openssl engine
In-Reply-To: <e6d3b076-b400-3be1-bf4a-a6d761978ff8@vaisala.com>
References: <20230712140707.364629-1-vesa.jaaskelainen@vaisala.com>
 <ZK9J6qdWPWYqSELG@mdounin.ru>
 <e6d3b076-b400-3be1-bf4a-a6d761978ff8@vaisala.com>
Message-ID: <ZLH5ciis7CWfWCwE@mdounin.ru>

Hello!

On Thu, Jul 13, 2023 at 10:38:02AM +0300, Vesa Jääskeläinen via nginx-devel wrote:

> Hi Maxim,
> 
> On 13.7.2023 3.48, Maxim Dounin wrote:
> > Hello!
> >
> > On Wed, Jul 12, 2023 at 05:07:03PM +0300, Vesa Jääskeläinen via nginx-devel wrote:
> >
> >> (I hope this goes properly out as I had major issues with hg email so
> >> combined hg export + git send-email)
> >>
> >> It is convenient to keep X.509 certificates related to key pairs stored in
> >> openssl engine within the engine.
> >>
> >> Implementation uses 'LOAD_CERT_CTRL' extension to fetch certificate from
> >> the engine. This extension is not supported by all engines and in those
> >> cases it should report with an error.
> >>
> >> Configuration is similar to what it is for 'ssl_certificate_key'.
> >>
> >> First certificate must match with ssl_certificate_key's key pair rest of
> >> the certificiates are added to the certificate chain.
> >>
> >> Example configuration with libp11's pkcs11 engine:
> >>
> >>    ssl_certificate      "engine:pkcs11:pkcs11:token=mytoken;object=mykey
> >>                          engine:pkcs11:pkcs11:token=mytoken;object=int-ca";
> >>    ssl_certificate_key  "engine:pkcs11:pkcs11:token=mytoken;object=mykey?pin-value=mypin";
> >>
> >> Tested the loading with two pkcs11 implementations SoftHSMv2 and with
> >> OP-TEE's PKCS11 Trusted Application running on Embedded Linux device.
> >>
> >> First three commits is the main beef and in order to make it more flexible
> >> added also last commit allowing intermediate certificates loaded from file
> >> system.
> >>
> >> Separator of space is used as there was already existing use of array for
> >> ssl_certificate configuration.
> > Just in case, a similar proposal was previously discussed here:
> >
> > https://mailman.nginx.org/pipermail/nginx-devel/2020-April/013130.html
> > https://mailman.nginx.org/pipermail/nginx-devel/2020-May/013142.html
> >
> > Notably, the review is here:
> >
> > https://mailman.nginx.org/pipermail/nginx-devel/2020-May/013152.html
> >
> > I'm additionally sceptical about this given that engine interface
> > is deprecated by OpenSSL.
> 
> Thanks for the links.
> 
> The provider interface in OpenSSL 3 is yet to be matured for PKCS11 usage.
> 
> libp11 so far seems to have stayed with "legacy" engine and just adapted 
> for OpenSSL 3.
> 
> Then there is new project that is still in development phase but looks 
> promising:
> 
> https://github.com/latchset/pkcs11-provider
> 
> But before that provider implementation matures and is available in 
> distributions it takes one or more major LTS releases.
> 
> While waiting for that it would be nice to have the support.
> 
> What is interesting that the CMD extension is already being integrated 
> to other software like curl, wpa-supplicant/hostapd, stunnel, ppp, 
> M2Crypto -- so I would say the feature is there to stay for a while. 
> Even not documented as official openssl "extension" it is there and is 
> being used.
> 
> Now openssl also has "store" API which can be extended with different 
> schemes too but in example libp11 does not today support that.
> 
> https://www.openssl.org/docs/man1.1.1/man7/ossl_store.html
> https://www.openssl.org/docs/man1.1.1/man3/OSSL_STORE_LOADER.html
> 
> Now the Nginx is already using the "legacy" interface so if it is enable 
> why not then support the certificate loading when it is available.

As pointed out in the review referenced above, the certificate 
loading in question is a limited interface specific to pkcs11 
engine only.  That is, it's not even the legacy interface, it's a 
non-portable extension to a legacy interface.

[...]

> With PEM files this is easy as one can craft them freely.

And that's the recommended approach.  Exporting appropriate 
certificates to a PEM file and using it in nginx configuration 
should be trivial enough.

>From security point of view there are no benefits in using engines 
to load certificates.

[...]

> Why I believe it is a good idea to support engine/or in future the 
> provider interface is that to keep the certificates in one place next to 
> the keys. With this if the key pair gets re-created or new certificates 
> to be updated if you have the labels there you just need to trigger the 
> refresh. Then there is no extra data to extract or such as then those 
> could be out-of-sync and people try to figure out problems with that.

Note that with the approach you suggest, re-creating certificates 
might anyway require editing nginx configuration, since 
certificate chain for the new certificate might be different and 
might require different number of certificates in the chain.

> If I am not mistaken the previous discussion comments has been addressed 
> in this change set.

As far as I see, you've tried to address the issue with 
certificate chains.  I can't say I like the approach you've taken 
though, as it basically introduced yet another array inside the 
ssl_certificate directive parameter.

> I am happy to test out the provider with pkcs11-provider too but I would 
> recommend that we also support today's methods so people can start using 
> the feature.

Thank you for your work, though I tend to think that introducing 
loading certificates from the pkcs11 engine is not something we 
are willing to do now, especially considering questionable 
approach to configuring things and the fact that engines are 
already deprecated.

Rather, I would consider introducing loading private keys and 
certificates via providers once this will be available.

-- 
Maxim Dounin
http://mdounin.ru/

From mdounin at mdounin.ru  Sat Jul 15 03:12:05 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Sat, 15 Jul 2023 06:12:05 +0300
Subject: Fixing the well Documented Nginx Alias Traversal Vulnerability?
In-Reply-To: <CAJq_=OVy9ZmhHfBWvbdA0pG6_QGB1jg_-LBHQsy_VcP+-Bt2cQ@mail.gmail.com>
References: <CAJq_=OXAOdULC4jnngiLJefcJNR6_hFrL0+34wAuCr-7W7hLkg@mail.gmail.com>
 <ZKieDyPi5c8PXdiP@mdounin.ru>
 <CAJq_=OU=0sWhvcwgPDAkRM8i7jhYFCx5GUS4Mt+QHWLDBZCGWQ@mail.gmail.com>
 <ZLAZ0FsBMrvGJEms@mdounin.ru>
 <CAJq_=OVy9ZmhHfBWvbdA0pG6_QGB1jg_-LBHQsy_VcP+-Bt2cQ@mail.gmail.com>
Message-ID: <ZLIOhVd2nkVpi7EN@mdounin.ru>

Hello!

On Fri, Jul 14, 2023 at 06:50:54PM -0400, Jonathan Leitschuh wrote:

> Hi Maxim,
> 
> > The "i.." string is a valid path component and there are no reasons why
> it shouldn't be allowed - there might be such files and/or directories on
> the disk.
> 
> Do you have any metrics on how often this is truly used? Or how common this
> file path really is? Is the argument here that "this might be valid, so we
> should support it" or is there some wider use case of this standard that
> I'm unfamiliar with? It seems puzzling to me to support a potential file
> name like "i.." if it's frequency of use is incredibly low, and, more often
> than not, likely to result in a security vulnerability.

As a general-purpose web server, we've seen multiple cases of 
perfectly invalid URIs, yet used in real life, and users asking 
us to support them.  Examples include spaces in URIs 
(https://trac.nginx.org/nginx/ticket/196) and URIs with invalid 
%xx encoding.

As for the names like "i..", a trivial github search is as 
follows:

https://github.com/search?q=path%3A..%2F&type=code

As of now it returns about 136k results.

> > Similarly, location matching and alias as implemented in nginx (and
> documented) are expected to result in "../app/config.py" appended to
> "/data/w3/images/" for such a configuration.
> 
> Speaking as a security researcher, this seems blatantly like behaviour that
> most people would never want intentionally. The only valid use cases I
> could see would be as an intentional vulnerability introduced by someone
> building an intentionally vulnerable application for something like a
> security capture-the-flag. But in those cases, the vulnerability is
> intentionally left in place, instead of accidentally introduced. I don't,
> personally, have any concerns with breaking those cases. Are you aware of
> any valid uses cases of this behaviour in the real world, or is this
> behaviour left because of hypothetical configurations that this may break?

I don't think there are configurations which specifically designed 
to work with "../" to allow addressing a directory above the alias 
specified.  Though it is important to keep in mind that this is 
just an edge case of a generic prefix matching behaviour.  Prefix 
matching, which is often used to match specific directories, such 
as in

    location /images/ {
        ...
    }

can be also used to match arbitrary prefixes instead, and a 
configuration like (which is outlined to be a vulnerable 
configuration in the article you've linked):

    location /images {
        ...
    }

can be intentionally used to match not only "/images", "/images/", 
"/images/foo.jpg", and so on, but also "/images2", "/images2/", 
"/images2/bar.jpg", and so on.  And certainly there are multiple 
such configurations in place.

Further, most of the configurations using prefix matching without 
trailing slash aren't vulnerable, even if matching of additional 
URIs is not intentional, because access to relevant resources is 
anyway allowed.

> 
> Cheers,
> Jonathan Leitschuh
> 
> 
> On Thu, Jul 13, 2023 at 11:35 AM Maxim Dounin <mdounin at mdounin.ru> wrote:
> 
> > Hello!
> >
> > On Wed, Jul 12, 2023 at 05:45:34PM -0400, Jonathan Leitschuh wrote:
> >
> > > Hi Maxim,
> > >
> > > Pleasure to meet you!
> > >
> > > I've also included Munawar, the CEO of Open Refactory in this email
> > chain.
> > >
> > > > (It looks like you aren't subscribed to nginx-devel@, so the message
> > did
> > > not reach the mailing list: it only accepts messages from subscribers.
> > > I've preserved it in Cc: though, so we can continue discussion there,
> > with
> > > more developers involved.)
> > >
> > > Do I need to subscribe to continue to get this thread included in that
> > > email group? The website didn't explicitly say that I needed to subscribe
> > > to send emails there, that might be a good addition.
> >
> > The http://nginx.org/en/support.html page says:
> >
> > "To post to a mailing list, an e-mail address that will be used
> > for posting must first be subscribed."
> >
> > This applies to all mailing lists on the page, including
> > nginx-devel at .  Sorry if it wasn't clear.
> >
> > > > Indeed, the misconfiguration in question is a well-known one. Gixy, a
> > > tool to detect various nginx misconfigurations, tries to detect it at
> > least
> > > since 2017:
> > > >
> > > >
> > >
> > https://github.com/yandex/gixy/blob/master/docs/en/plugins/aliastraversal.md
> > >
> > > From the documentation, they provide the following example request:
> > > `/i../app/config.py`. I can't imagine a case where a request like
> > > `/i../app/config.py` could ever be made legitimately by a user. Why is
> > > `../app/config.py` appended to the alias `/data/w3/images/` in this case
> > > without nginx throwing an exception as this is a blatant path traversal
> > > attack?
> >
> > The "i.." string is a valid path component and there are no
> > reasons why it shouldn't be allowed - there might be such files
> > and/or directories on the disk.  Similarly, location matching
> > and alias as implemented in nginx (and documented) are expected to
> > result in "../app/config.py" appended to "/data/w3/images/" for
> > such a configuration.
> >
> > The question here is why the location prefix without trailing "/"
> > was used in the configuration, and if it was intentional.
> >
> > > > Unfortunately, I don't think there is an easy fix.  Changing the way
> > how
> > > nginx locations work would be a major change, and will break a lot of
> > valid
> > > configurations where prefix matching is intentionally used.
> > >
> > > This sounds like a fix that could this go through a depreciation cycle,
> > > where, nginx adds an intentional, opt-in, prefix-matching flag that, in
> > > it's early incarnations, offers warnings to users for
> > > deprecated configuration, and later, will fail-closed unless the flag is
> > > present?
> >
> > Sure, major changes can be implemented in a way which warns users
> > in advance.  This still be a major change though, and the
> > transition will require several years and will disturb a lot of
> > users, including ones who aren't affected by the configuration
> > issue in question.
> >
> > Further, it is not clear if it will help: nothing will stop users
> > from using the explicit flag for prefix matching in an unsafe way
> > without realizing it's unsafe.
> >
> > Given that, an explicit warning/note in the documentation looks
> > like the way to go, at least for now.
> >
> > >
> > > > If you think there is an easy fix, feel free to provide suggestions.
> > >
> > > On our side, between Munwar and myself, we have some significant
> > experience
> > > with bulk pull request generation to fix security vulnerabilities
> > at-scale
> > > across open source software. (See my Black Hat & DEF CON Talk on the
> > topic
> > > from last year
> > > <
> > https://github.com/jlleitschuh/#scaling-the-security-researcher-to-eliminate-oss-vulnerabilities-once-and-for-all
> > >
> > > )
> > >
> > > Munawar and his team at OpenRefactory did an analysis of the 2k+
> > > repositories that GitHub flags with the above linked search. From his
> > > team's analysis:
> > >
> > >  - Majority of the bugs are in documentation. These can be easily
> > filtered
> > > out with the file type. We can consider bulk pull request generation
> > adding
> > > a bit that we are fixing a documentation to make it correct so that
> > people
> > > are not misinformed and left with a vulnerability.
> > >  - There are a few bugs in the actual config files. In such cases, you
> > may
> > > split the projects based on the GH stars. The more popular ones should
> > get
> > > a stronger message communicating the risk to their end-users.
> > >
> > > We can likely massively clean-up this vulnerability across the OSS
> > > ecosystem, but the long-term solution to this vulnerability, I believe,
> > > will lie in the hands of the Nginx team.
> > >
> > > How can we work together here towards a world where this vulnerability is
> > > something end-users opt-in to, with intention (usually via some flag),
> > > rather than the world we are currently in, where this vulnerability
> > appears
> > > as a mistake?
> > >
> > > Cheers,
> > > Jonathan Leitschuh
> > >
> > >
> > > On Fri, Jul 7, 2023 at 7:21 PM Maxim Dounin <mdounin at mdounin.ru> wrote:
> > >
> > > > Hello!
> > > >
> > > > On Thu, Jul 06, 2023 at 12:05:25PM -0400, Jonathan Leitschuh wrote:
> > > >
> > > > > Hi Nginx Team,
> > > > >
> > > > > My name is Jonathan Leitschuh. I'm a Senior Software Security
> > Researcher
> > > > > for the Open Source Security Foundation: Project Alpha-Omega
> > > > > <https://openssf.org/community/alpha-omega/>. “Alpha” works with the
> > > > > maintainers of the most critical open source projects to help them
> > > > identify
> > > > > and fix security vulnerabilities, and improve their security posture.
> > > > > “Omega” identified at least 10,000 widely deployed OSS projects
> > where it
> > > > > can apply automated security analysis, scoring, and remediation
> > guidance
> > > > to
> > > > > their open source maintainer communities.
> > > > >
> > > > > Earlier today, on Twitter, I saw this discussion regarding
> > > > > the long-standing the NGinx Alias traversal vulnerability:
> > > > > https://twitter.com/infosec_au/status/1676072447156834304?s=20
> > > > >
> > > > > In particular, this discussion centered on this, this recent blog
> > post
> > > > > published on the 3rd of this month.
> > > > > https://labs.hakaioffsec.com/nginx-alias-traversal/
> > > > >
> > > > > In that blog post, the researchers detail how, due to
> > misconfiguration
> > > > and
> > > > > failing to specify the `/` character in the correct place can lead to
> > > > path
> > > > > traversal vulnerabilities.
> > > > >
> > > > >
> > > > >
> > > > > This vulnerability was previously discussed by Detectify in a blog
> > post
> > > > in
> > > > > 2020:
> > > > >
> > https://blog.detectify.com/2020/11/10/common-nginx-misconfigurations/
> > > > >
> > > > > This research originated with Orange Tsai back in 2018:
> > > > >
> > > >
> > https://i.blackhat.com/us-18/Wed-August-8/us-18-Orange-Tsai-Breaking-Parser-Logic-Take-Your-Path-Normalization-Off-And-Pop-0days-Out-2.pdf
> > > > >
> > > > > The recent hakaioffsec blog also identifies this case where the
> > > > > vulnerability also appears:
> > > > >
> > > > >
> > > > > My question to both the Nginx Security team, and the Nginx
> > Developers,
> > > > why
> > > > > not fix this vulnerability in Nginx itself such that it's safe by
> > > > default,
> > > > > but users can opt-in to the vulnerable behaviour if they so desire
> > > > through
> > > > > an explicit flag? Surely it would be fairly straightforward to
> > identify
> > > > > these vulnerable configurations and fix them for the user, or fail
> > closed
> > > > > (safe)?
> > > > >
> > > > > If you need evidence of the impact this vulnerability can have, the
> > > > > above hakaioffsec blog post details how this vulnerability resulted
> > in
> > > > > a US$6000 bounty from BitWarden.
> > > > >
> > > > > This GitHub search, proposed by the above blog post, returns 2.2k
> > results
> > > > > on GitHub:
> > > > >
> > > > >
> > > >
> > https://github.com/search?q=%2Flocation+%5C%2F%5B_.a-zA-Z0-9-%5C%2F%5D*%5B%5E%5C%2F%5D%5B%5Cs%5D%5C%7B%5B%5Cs%5Cn%5D*alias+%5C%2F%5B_.a-zA-Z0-9-%5C%2F%5D*%5C%2F%3B%2F+&type=code
> > > > >
> > > > > I'm curious what is currently preventing this from being fixed
> > upstream
> > > > to
> > > > > prevent end-users from accidentally implementing vulnerable
> > > > configurations
> > > > > like this.
> > > >
> > > > Thank you for your message.
> > > >
> > > > Indeed, the misconfiguration in question is a well-known one.
> > > > Gixy, a tool to detect various nginx misconfigurations, tries to
> > > > detect it at least since 2017:
> > > >
> > > >
> > > >
> > https://github.com/yandex/gixy/blob/master/docs/en/plugins/aliastraversal.md
> > > >
> > > > The root cause as I see it is that nginx locations (and alias)
> > > > work with prefix strings, while unaware people often expect them
> > > > to work with path components (directories) instead.  Note the
> > > > "Achieving Impact Without a Slash on Alias" section in the article
> > > > you've referenced: a configuration like
> > > >
> > > >     location /img {
> > > >         alias /var/images;
> > > >     }
> > > >
> > > > makes it possible to access "/var/images_confidential" via a
> > > > request to "/img_confidential".  This is because locations works
> > > > on prefix strings, and due to lack of trailing "/" in the location
> > > > prefix it does match "/img_confidential" request, and maps it to
> > > > the file system according to the configuration.
> > > >
> > > > Similarly, the same misconfiguration can happen even without the
> > > > "alias" directive at all. Consider the configuration:
> > > >
> > > >     location / { deny all; }
> > > >     location /images { allow all; }
> > > >
> > > > In contrast to some might (incorrectly) assume, a request to
> > > > "/images_confidential" will be allowed, and will return content as
> > > > per the configuration.
> > > >
> > > > Unfortunately, I don't think there is an easy fix.  Changing the
> > > > way how nginx locations work would be a major change, and will
> > > > break a lot of valid configurations where prefix matching is
> > > > intentionally used.
> > > >
> > > > Meanwhile, it surely deserves some additional warnings/notes in
> > > > the documentation, to emphasize the actual behaviour and potential
> > > > pitfalls.  Hopefully it would be enough to at least reduce the
> > > > amount of such misconfigurations.
> > > >
> > > > If you think there is an easy fix, feel free to provide
> > > > suggestions.
> > > >
> > > > (It looks like you aren't subscribed to nginx-devel@, so the
> > > > message did not reach the mailing list: it only accepts messages
> > > > from subscribers.  I've preserved it in Cc: though, so we can
> > > > continue discussion there, with more developers involved.)
> > > >
> > > > --
> > > > Maxim Dounin
> > > > http://mdounin.ru/
> > > >
> >
> > --
> > Maxim Dounin
> > http://mdounin.ru/
> >

-- 
Maxim Dounin
http://mdounin.ru/

From jordanc.carter at outlook.com  Sat Jul 15 03:48:25 2023
From: jordanc.carter at outlook.com (J Carter)
Date: Sat, 15 Jul 2023 03:48:25 +0000
Subject: [PATCH] Added debug_random directive
Message-ID: <LO0P123MB63194D794B1808A023A7FC9C9E35A@LO0P123MB6319.GBRP123.PROD.OUTLOOK.COM>

# HG changeset patch
# User J Carter <jordanc.carter at outlook.com>
# Date 1689391559 -3600
#      Sat Jul 15 04:25:59 2023 +0100
# Node ID b1ea0a60417e547513654bf9d6bb79714865c780
# Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
Added debug_random directive

This directive enforces for EITHER a percentage of total connections
OR a percentage of connections matched by debug_connection CIDRs
to have debug logging enabled.

This is useful for debugging when nginx is under high load
(production) - where debugging all connections is not possible without
disrupting traffic.

This directive takes a value between 0.00%-100.00% exclusive.

Example usage:

events {
    worker_connections  1024;
    #if uncommented, the percentage applies to connection from lo.
    #debug_connection    127.0.0.1/8;
    debug_random        1%;
}

diff -r 77c1418916f7 -r b1ea0a60417e src/event/ngx_event.c
--- a/src/event/ngx_event.c	Thu Jun 08 14:58:01 2023 +0400
+++ b/src/event/ngx_event.c	Sat Jul 15 04:25:59 2023 +0100
@@ -30,6 +30,8 @@
 static char *ngx_event_use(ngx_conf_t *cf, ngx_command_t *cmd, void
*conf); static char *ngx_event_debug_connection(ngx_conf_t *cf,
ngx_command_t *cmd, void *conf);
+static char *ngx_event_debug_random(ngx_conf_t *cf, ngx_command_t *cmd,
+    void *conf);
 
 static void *ngx_event_core_create_conf(ngx_cycle_t *cycle);
 static char *ngx_event_core_init_conf(ngx_cycle_t *cycle, void *conf);
@@ -162,6 +164,13 @@
       0,
       NULL },
 
+      { ngx_string("debug_random"),
+      NGX_EVENT_CONF|NGX_CONF_TAKE1,
+      ngx_event_debug_random,
+      0,
+      0,
+      NULL },
+
       ngx_null_command
 };
 
@@ -1254,6 +1263,53 @@
 }
 
 
+static char *
+ngx_event_debug_random(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+#if (NGX_DEBUG)
+    ngx_event_conf_t  *ecf = conf;
+
+    u_char               *c;
+    ngx_int_t             pct;
+    ngx_uint_t            len;
+    ngx_str_t            *value;
+
+    value = cf->args->elts;
+    c = value[1].data;
+    len = ngx_strlen(c);
+
+    if (c[len-1] != '%') {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                               "%V missing '%'",
+                               &value[1]);
+        return NGX_CONF_ERROR;
+    }
+
+    pct = ngx_atofp(c, len-1, 2);
+
+    if (pct == NGX_ERROR
+        || pct == 0
+        || pct > 9999) {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                               "%V is an invalid value",
+                               &value[1]);
+        return NGX_CONF_ERROR;
+    }
+
+    ecf->debug_rnd = NGX_MAX_INT32_VALUE / 10000 * pct;
+
+#else
+
+    ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
+                       "\"debug_random\" is ignored, you need to
rebuild "
+                       "nginx using --with-debug option to enable it");
+
+#endif
+
+    return NGX_CONF_OK;
+}
+
+
 static void *
 ngx_event_core_create_conf(ngx_cycle_t *cycle)
 {
diff -r 77c1418916f7 -r b1ea0a60417e src/event/ngx_event.h
--- a/src/event/ngx_event.h	Thu Jun 08 14:58:01 2023 +0400
+++ b/src/event/ngx_event.h	Sat Jul 15 04:25:59 2023 +0100
@@ -438,6 +438,7 @@
     u_char       *name;
 
 #if (NGX_DEBUG)
+    ngx_uint_t    debug_rnd;
     ngx_array_t   debug_connection;
 #endif
 } ngx_event_conf_t;
diff -r 77c1418916f7 -r b1ea0a60417e src/event/ngx_event_accept.c
--- a/src/event/ngx_event_accept.c	Thu Jun 08 14:58:01 2023 +0400
+++ b/src/event/ngx_event_accept.c	Sat Jul 15 04:25:59 2023 +0100
@@ -523,6 +523,14 @@
     struct sockaddr_in6  *sin6;
     ngx_uint_t            n;
 #endif
+    ngx_uint_t r = ngx_random();
+
+    if (!ecf->debug_connection.nelts) {
+        c->log->log_level = (r < ecf->debug_rnd) ?
+                            NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL
:
+                            c->log->log_level;
+        return;
+    }
 
     cidr = ecf->debug_connection.elts;
     for (i = 0; i < ecf->debug_connection.nelts; i++) {
@@ -561,7 +569,9 @@
             break;
         }
 
-        c->log->log_level = NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL;
+        c->log->log_level = (r < ecf->debug_rnd) ?
+                            NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL:
:
+                            c->log->log_level;
         break;
 
     next:

From mdounin at mdounin.ru  Tue Jul 18 00:19:04 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 18 Jul 2023 03:19:04 +0300
Subject: [PATCH] Core: fixed environment on exit better
In-Reply-To: <74BC1CA2-9522-43AA-9EFD-24581BF6F3FB@nginx.com>
References: <34a8e1a4161542896c11.1687535747@enoparse.local>
 <ZJZPtrxW0uDtw/57@mdounin.ru>
 <74BC1CA2-9522-43AA-9EFD-24581BF6F3FB@nginx.com>
Message-ID: <ZLXaeGERpd5BcoQz@mdounin.ru>

Hello!

On Thu, Jul 13, 2023 at 08:58:44PM +0400, Sergey Kandaurov wrote:

> > On 24 Jun 2023, at 06:06, Maxim Dounin <mdounin at mdounin.ru> wrote:
> > 
> > Hello!
> > 
> > On Fri, Jun 23, 2023 at 07:55:47PM +0400, Sergey Kandaurov wrote:
> > 
> >> # HG changeset patch
> >> # User Sergey Kandaurov <pluknet at nginx.com>
> >> # Date 1687535739 -14400
> >> #      Fri Jun 23 19:55:39 2023 +0400
> >> # Node ID 34a8e1a4161542896c11c4a5c60d6a6fe1931e3d
> >> # Parent  c681f4906d838ac45b517fada4c4274ade03fd3c
> >> Core: fixed environment on exit better.
> >> 
> >> The fix in 6822:c045b4926b2c to replace environment allocations from a pool
> >> was incomplete, such that it left pool owned allocations for variables set
> >> with values using the "env" directive.  Another consumer of this interface
> >> is the QUIC BPF helper that uses the environment variable NGINX_BPF_MAPS.
> >> The fix is to re-allocate variables with malloc, to avoid use after free.
> >> If memory allocation failed, the only option is to expel from environment.
> >> 
> >> The observed symptoms are similar to described in 6822:c045b4926b2c, that
> >> is a segmentation fault on worker process exit from atexit() handler seen
> >> with 3rd party modules or if nginx was built with lcov profiling.
> >> 
> >> diff --git a/src/core/nginx.c b/src/core/nginx.c
> >> --- a/src/core/nginx.c
> >> +++ b/src/core/nginx.c
> >> @@ -631,17 +631,48 @@ ngx_cleanup_environment(void *data)
> >> {
> >>     char  **env = data;
> >> 
> >> -    if (environ == env) {
> >> +    char        *p, **penv;
> >> +    size_t       size;
> >> +    ngx_uint_t   i, n;
> >> 
> >> -        /*
> >> -         * if the environment is still used, as it happens on exit,
> >> -         * the only option is to leak it
> >> -         */
> >> -
> >> +    if (environ != env) {
> >> +        ngx_free(env);
> >>         return;
> >>     }
> > 
> > Note: if we assume that the environment can be arbitrary modified, 
> > I don't think it's enough.  For example, a setenv() call can 
> > replace environ with something different, so this code will free 
> > env and exit, but pool-allocated variables will be still 
> > referenced by the new environ.
> > 
> > This is probably out of the scope of this patch though.
> 
> It could be solved by always checking for pool-allocated variables,
> irrespective of whether environ was changed after setenv(),
> using that as an additional condition to free environment.
> 
> > 
> >> 
> >> -    ngx_free(env);
> >> +    /*
> >> +     * if the environment is still used, as it happens on exit,
> >> +     * the only option is to leak it
> >> +     */
> >> +
> >> +    for (n = 0; env[n]; n++) { }
> >> +
> >> +    for (i = 0; env[i]; i++) {
> >> +
> >> +        for (penv = ngx_os_environ; *penv; penv++) {
> >> +            if (ngx_strcmp(*penv, env[i]) == 0) {
> > 
> > I don't think it's going to work.  If I'm reading this correctly, 
> > with the initial environment "FOO=bar" and "env FOO=bar;" in the 
> > configuration this will assume the string is from ngx_os_environ, 
> > but it will be from ccf->env instead.
> > 
> 
> Thanks for catching that.
> So, we have to compare against ccf->env, instead.
> 
> >> +                break;
> >> +            }
> >> +        }
> >> +
> >> +        if (*penv) {
> >> +            continue;
> >> +        }
> >> +
> >> +        /* substitute pool allocated variables */
> >> +
> >> +        size = ngx_strlen(env[i]) + 1;
> >> +        p = malloc(size);
> > 
> > There should be no direct malloc() calls except in low-level code 
> > where logging is not available.  I don't think this particular 
> > place counts, so ngx_alloc() should be used instead.
> 
> I used to avoid using cycle in cleanup, thus direct malloc() calls.
> Anyway, it will be needed in order to access configuration.
> 
> Below is an updated patch (commit log unchanged).
> 
> # HG changeset patch
> # User Sergey Kandaurov <pluknet at nginx.com>
> # Date 1689267366 -14400
> #      Thu Jul 13 20:56:06 2023 +0400
> # Node ID 80a60d9707ed4b80402335d3d31a978f418f8056
> # Parent  f91dc350be9f4a6bf1379a32a210afece7b0d75e
> Core: fixed environment on exit better.
> 
> The fix in 6822:c045b4926b2c to replace environment allocations from a pool
> was incomplete, such that it left pool owned allocations for variables set
> with values using the "env" directive.  Another consumer of this interface
> is the QUIC BPF helper that uses the environment variable NGINX_BPF_MAPS.
> The fix is to re-allocate variables with malloc, to avoid use after free.
> If memory allocation failed, the only option is to expel from environment.
> 
> The observed symptoms are similar to described in 6822:c045b4926b2c, that
> is a segmentation fault on worker process exit from atexit() handler seen
> with 3rd party modules or if nginx was built with lcov profiling.
> 
> diff --git a/src/core/nginx.c b/src/core/nginx.c
> --- a/src/core/nginx.c
> +++ b/src/core/nginx.c
> @@ -592,7 +592,7 @@ tz_found:
>          }
>  
>          cln->handler = ngx_cleanup_environment;
> -        cln->data = env;
> +        cln->data = cycle;
>      }
>  
>      n = 0;
> @@ -629,19 +629,57 @@ tz_found:
>  static void
>  ngx_cleanup_environment(void *data)
>  {
> -    char  **env = data;
> -
> -    if (environ == env) {
> +    ngx_cycle_t  *cycle = data;
>  
> -        /*
> -         * if the environment is still used, as it happens on exit,
> -         * the only option is to leak it
> -         */
> +    char             *p, **env;
> +    size_t            size;
> +    ngx_str_t        *var;
> +    ngx_uint_t        i, n;
> +    ngx_core_conf_t  *ccf;
>  
> +    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
> +    env = ccf->environment;
> +
> +    if (environ != env) {
> +        ngx_free(env);
>          return;
>      }
>  
> -    ngx_free(env);
> +    /*
> +     * if the environment is still used, as it happens on exit,
> +     * the only option is to leak it
> +     */
> +
> +    for (n = 0; env[n]; n++) { /* void */ }
> +
> +    while (*env) {
> +        var = ccf->env.elts;
> +
> +        for (i = 0; i < ccf->env.nelts; i++) {
> +            if (var[i].data == (u_char *) *env) {
> +                break;
> +            }
> +        }
> +
> +        if (i == ccf->env.nelts) {
> +            env++;
> +            continue;
> +        }
> +
> +        /* substitute pool allocated variables */
> +
> +        size = ngx_strlen(*env) + 1;
> +        p = ngx_alloc(size, cycle->log);
> +
> +        if (p) {
> +            ngx_memcpy(p, *env, size);
> +            *env++ = p;
> +
> +        } else {
> +            ngx_memmove(env, env + 1, sizeof(char *) * (n - (env - environ)));
> +            n--;
> +        }
> +    }
>  }
>  
>  
> 
> > 
> >> +
> >> +        if (p) {
> >> +            ngx_memcpy(p, env[i], size);
> >> +            env[i] = p;
> >> +
> >> +        } else {
> >> +            ngx_memmove(&env[i], &env[i + 1], sizeof(char *) * (n - i));
> >> +            i--; n--;
> >> +        }
> >> +    }
> >> }
> > 
> > Overall, it looks like a better idea might be to simply add all 
> > the variables added to the environment to the environ allocation 
> > in ngx_set_environment().
> > 
> > In ngx_set_environment() we explicitly know if the particular 
> > variable is from OS environment or from ccf->environ, so there 
> > will be no need to guess.  Also, the appropriate log is readily 
> > available (and already used in the existing ngx_alloc() call), and 
> > the code to cleanup things is already there and will work without 
> > additional modifications.
> 
> I tried to avoid allocations in ngx_set_environment(),
> as it can be called from the master process and harm
> hypothetically by accumulating leaked allocations.

The ngx_set_environment() function installs a cleanup handler, 
which is to be called on cycle pool destruction, and therefore 
there can't be a leak unless the cleanup handler will decide it 
needs to leak the allocation.

And, if the cleanup handler will decide to leak it, there should 
be no difference with re-allocating things in the cleanup handler 
(except the fact that allocations in cleanup handler are risky, 
see below).

Rather, I would prefer to avoid allocations in 
ngx_cleanup_environment(), since on failure this will corrupt the 
environment with unpredictable results, and there are no options 
to avoid this.  Allocating everything in ngx_set_environment() 
seems to be much safer approach.

I've tried a few variants, and the most simple one seems to 
allocate variables one-by-one with appropriate cleanup handlers in 
ngx_set_environment().  (Doing the same in ngx_set_env() is 
slightly easier, but doesn't play well with QUIC.)  This approach 
addresses all possible environment modifications and also looks 
perfectly in line with the existing code.

Here is the patch:

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1689637083 -10800
#      Tue Jul 18 02:38:03 2023 +0300
# Node ID dedb235210f816afc028f60a525009fc5dabd8bf
# Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
Core: fixed environment variables on exit.

Similarly to 6822:c045b4926b2c, environment variables introduced with
the "env" directive (and "NGINX_BPF_MAPS" added by QUIC) are now allocated
via ngx_alloc(), and explicitly freed by a cleanup handler if no longer used.

In collaboration with Sergey Kandaurov.

diff --git a/src/core/nginx.c b/src/core/nginx.c
--- a/src/core/nginx.c
+++ b/src/core/nginx.c
@@ -13,6 +13,7 @@
 static void ngx_show_version_info(void);
 static ngx_int_t ngx_add_inherited_sockets(ngx_cycle_t *cycle);
 static void ngx_cleanup_environment(void *data);
+static void ngx_cleanup_environment_variable(void *data);
 static ngx_int_t ngx_get_options(int argc, char *const *argv);
 static ngx_int_t ngx_process_options(ngx_cycle_t *cycle);
 static ngx_int_t ngx_save_argv(ngx_cycle_t *cycle, int argc, char *const *argv);
@@ -518,7 +519,8 @@ ngx_add_inherited_sockets(ngx_cycle_t *c
 char **
 ngx_set_environment(ngx_cycle_t *cycle, ngx_uint_t *last)
 {
-    char                **p, **env;
+    char                **p, **env, *str;
+    size_t                len;
     ngx_str_t            *var;
     ngx_uint_t            i, n;
     ngx_core_conf_t      *ccf;
@@ -600,7 +602,31 @@ tz_found:
     for (i = 0; i < ccf->env.nelts; i++) {
 
         if (var[i].data[var[i].len] == '=') {
-            env[n++] = (char *) var[i].data;
+
+            if (last) {
+                env[n++] = (char *) var[i].data;
+                continue;
+            }
+
+            cln = ngx_pool_cleanup_add(cycle->pool, 0);
+            if (cln == NULL) {
+                return NULL;
+            }
+
+            len = ngx_strlen(var[i].data) + 1;
+
+            str = ngx_alloc(len, cycle->log);
+            if (str == NULL) {
+                return NULL;
+            }
+
+            ngx_memcpy(str, var[i].data, len);
+
+            cln->handler = ngx_cleanup_environment_variable;
+            cln->data = str;
+
+            env[n++] = str;
+
             continue;
         }
 
@@ -645,6 +671,29 @@ ngx_cleanup_environment(void *data)
 }
 
 
+static void
+ngx_cleanup_environment_variable(void *data)
+{
+    char  *var = data;
+
+    char  **p;
+
+    for (p = environ; *p; p++) {
+
+        /*
+         * if an environment variable is still used, as it happens on exit,
+         * the only option is to leak it
+         */
+
+        if (*p == var) {
+            return;
+        }
+    }
+
+    ngx_free(var);
+}
+
+
 ngx_pid_t
 ngx_exec_new_binary(ngx_cycle_t *cycle, char *const *argv)
 {


-- 
Maxim Dounin
http://mdounin.ru/

From jordanc.carter at outlook.com  Tue Jul 18 03:36:02 2023
From: jordanc.carter at outlook.com (J Carter)
Date: Tue, 18 Jul 2023 03:36:02 +0000
Subject: [PATCH] Added debug_random directive
In-Reply-To: <LO0P123MB63194D794B1808A023A7FC9C9E35A@LO0P123MB6319.GBRP123.PROD.OUTLOOK.COM>
References: <LO0P123MB63194D794B1808A023A7FC9C9E35A@LO0P123MB6319.GBRP123.PROD.OUTLOOK.COM>
Message-ID: <LO0P123MB63193421DA656E482B9357FF9E38A@LO0P123MB6319.GBRP123.PROD.OUTLOOK.COM>

On Sat, 15 Jul 2023 03:48:25 +0000
J Carter <jordanc.carter at outlook.com> wrote:

> # HG changeset patch
> # User J Carter <jordanc.carter at outlook.com>
> # Date 1689391559 -3600
> #      Sat Jul 15 04:25:59 2023 +0100
> # Node ID b1ea0a60417e547513654bf9d6bb79714865c780
> # Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
> Added debug_random directive
> 
> This directive enforces for EITHER a percentage of total connections
> OR a percentage of connections matched by debug_connection CIDRs
> to have debug logging enabled.
> 
> This is useful for debugging when nginx is under high load
> (production) - where debugging all connections is not possible without
> disrupting traffic.
> 
> This directive takes a value between 0.00%-100.00% exclusive.
> 

# HG changeset patch
# User J Carter <jordanc.carter at outlook.com>
# Date 1689649226 -3600
#      Tue Jul 18 04:00:26 2023 +0100
# Node ID 87f6f95e0385e6cd37354979ea61cc2435deb430
# Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
Added debug_random directive

Rework of previous patch. Fixed several bugs.

Example usage:

events {
     worker_connections  1024;
     #if uncommented, the percentage applies to connection from lo.
     #debug_connection    127.0.0.0/8;
     debug_random        1%;
}

diff -r 77c1418916f7 -r 87f6f95e0385 src/event/ngx_event.c
--- a/src/event/ngx_event.c	Thu Jun 08 14:58:01 2023 +0400
+++ b/src/event/ngx_event.c	Tue Jul 18 04:00:26 2023 +0100
@@ -30,6 +30,8 @@
 static char *ngx_event_use(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
 static char *ngx_event_debug_connection(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+static char *ngx_event_debug_random(ngx_conf_t *cf, ngx_command_t *cmd,
+    void *conf);
 
 static void *ngx_event_core_create_conf(ngx_cycle_t *cycle);
 static char *ngx_event_core_init_conf(ngx_cycle_t *cycle, void *conf);
@@ -162,6 +164,13 @@
       0,
       NULL },
 
+      { ngx_string("debug_random"),
+      NGX_EVENT_CONF|NGX_CONF_TAKE1,
+      ngx_event_debug_random,
+      0,
+      0,
+      NULL },
+
       ngx_null_command
 };
 
@@ -496,6 +505,7 @@
     size_t               size, cl;
     ngx_shm_t            shm;
     ngx_time_t          *tp;
+    ngx_cidr_t          *cidr;
     ngx_core_conf_t     *ccf;
     ngx_event_conf_t    *ecf;
 
@@ -507,6 +517,29 @@
                       "using the \"%s\" event method", ecf->name);
     }
 
+    if (ecf->debug_connection.nelts == 0 && ecf->debug_rnd > 0) {
+        cidr = ngx_array_push(&ecf->debug_connection);
+        if (cidr == NULL) {
+            return NGX_ERROR;
+        }
+        /*0.0.0.0/0*/
+        ngx_memzero(cidr, sizeof(ngx_cidr_t));
+        cidr->family = AF_INET;
+
+#ifdef NGX_HAVE_INET6
+        cidr = ngx_array_push(&ecf->debug_connection);
+        if (cidr == NULL) {
+            return NGX_ERROR;
+        }
+        /*::/0*/
+        ngx_memzero(cidr, sizeof(ngx_cidr_t));
+        cidr->family = AF_INET6;
+#endif
+
+    } else if (ecf->debug_connection.nelts > 0 && ecf->debug_rnd == 0) {
+        ecf->debug_rnd = NGX_MAX_INT32_VALUE;
+    }
+
     ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
 
     ngx_timer_resolution = ccf->timer_resolution;
@@ -1254,6 +1287,54 @@
 }
 
 
+static char *
+ngx_event_debug_random(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+#if (NGX_DEBUG)
+    ngx_event_conf_t  *ecf = conf;
+
+    u_char               *c;
+    ngx_int_t             pct;
+    ngx_uint_t            len;
+    ngx_str_t            *value;
+
+    value = cf->args->elts;
+    c = value[1].data;
+    len = value[1].len;
+
+    if (c[len-1] != '%') {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                               "%V missing '%'",
+                               &value[1]);
+        return NGX_CONF_ERROR;
+    }
+
+    pct = ngx_atofp(c, len-1, 2);
+
+    if (pct == NGX_ERROR
+        || pct == 0
+        || pct > 9999)
+    {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                               "%V is an invalid value",
+                               &value[1]);
+        return NGX_CONF_ERROR;
+    }
+
+    ecf->debug_rnd = NGX_MAX_INT32_VALUE / 10000 * pct;
+
+#else
+
+    ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
+                       "\"debug_random\" is ignored, you need to rebuild "
+                       "nginx using --with-debug option to enable it");
+
+#endif
+
+    return NGX_CONF_OK;
+}
+
+
 static void *
 ngx_event_core_create_conf(ngx_cycle_t *cycle)
 {
@@ -1279,6 +1360,8 @@
         return NULL;
     }
 
+    ecf->debug_rnd = NGX_CONF_UNSET_UINT;
+
 #endif
 
     return ecf;
@@ -1369,5 +1452,7 @@
     ngx_conf_init_value(ecf->accept_mutex, 0);
     ngx_conf_init_msec_value(ecf->accept_mutex_delay, 500);
 
+    ngx_conf_init_uint_value(ecf->debug_rnd, 0);
+
     return NGX_CONF_OK;
 }
diff -r 77c1418916f7 -r 87f6f95e0385 src/event/ngx_event.h
--- a/src/event/ngx_event.h	Thu Jun 08 14:58:01 2023 +0400
+++ b/src/event/ngx_event.h	Tue Jul 18 04:00:26 2023 +0100
@@ -438,6 +438,7 @@
     u_char       *name;
 
 #if (NGX_DEBUG)
+    ngx_uint_t    debug_rnd;
     ngx_array_t   debug_connection;
 #endif
 } ngx_event_conf_t;
diff -r 77c1418916f7 -r 87f6f95e0385 src/event/ngx_event_accept.c
--- a/src/event/ngx_event_accept.c	Thu Jun 08 14:58:01 2023 +0400
+++ b/src/event/ngx_event_accept.c	Tue Jul 18 04:00:26 2023 +0100
@@ -523,6 +523,7 @@
     struct sockaddr_in6  *sin6;
     ngx_uint_t            n;
 #endif
+    ngx_uint_t r = ngx_random();
 
     cidr = ecf->debug_connection.elts;
     for (i = 0; i < ecf->debug_connection.nelts; i++) {
@@ -561,7 +562,9 @@
             break;
         }
 
-        c->log->log_level = NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL;
+        c->log->log_level = (r < ecf->debug_rnd) ?
+                            NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL:
+                            c->log->log_level;
         break;
 
     next:

From pluknet at nginx.com  Tue Jul 18 17:51:49 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Tue, 18 Jul 2023 21:51:49 +0400
Subject: [PATCH] Core: fixed environment on exit better
In-Reply-To: <ZLXaeGERpd5BcoQz@mdounin.ru>
References: <34a8e1a4161542896c11.1687535747@enoparse.local>
 <ZJZPtrxW0uDtw/57@mdounin.ru>
 <74BC1CA2-9522-43AA-9EFD-24581BF6F3FB@nginx.com>
 <ZLXaeGERpd5BcoQz@mdounin.ru>
Message-ID: <BEA3FB1D-0140-4FC7-A0F5-2B2EDA5C510F@nginx.com>


> On 18 Jul 2023, at 04:19, Maxim Dounin <mdounin at mdounin.ru> wrote:
> 
> Hello!
> 
> On Thu, Jul 13, 2023 at 08:58:44PM +0400, Sergey Kandaurov wrote:
> 
>>> On 24 Jun 2023, at 06:06, Maxim Dounin <mdounin at mdounin.ru> wrote:
>>> 
>>> Hello!
>>> 
>>> On Fri, Jun 23, 2023 at 07:55:47PM +0400, Sergey Kandaurov wrote:
>>> 
>>>> # HG changeset patch
>>>> # User Sergey Kandaurov <pluknet at nginx.com>
>>>> # Date 1687535739 -14400
>>>> #      Fri Jun 23 19:55:39 2023 +0400
>>>> # Node ID 34a8e1a4161542896c11c4a5c60d6a6fe1931e3d
>>>> # Parent  c681f4906d838ac45b517fada4c4274ade03fd3c
>>>> Core: fixed environment on exit better.
>>>> 
>>>> The fix in 6822:c045b4926b2c to replace environment allocations from a pool
>>>> was incomplete, such that it left pool owned allocations for variables set
>>>> with values using the "env" directive.  Another consumer of this interface
>>>> is the QUIC BPF helper that uses the environment variable NGINX_BPF_MAPS.
>>>> The fix is to re-allocate variables with malloc, to avoid use after free.
>>>> If memory allocation failed, the only option is to expel from environment.
>>>> 
>>>> The observed symptoms are similar to described in 6822:c045b4926b2c, that
>>>> is a segmentation fault on worker process exit from atexit() handler seen
>>>> with 3rd party modules or if nginx was built with lcov profiling.
>>>> 
>>>> diff --git a/src/core/nginx.c b/src/core/nginx.c
>>>> --- a/src/core/nginx.c
>>>> +++ b/src/core/nginx.c
>>>> @@ -631,17 +631,48 @@ ngx_cleanup_environment(void *data)
>>>> {
>>>>    char  **env = data;
>>>> 
>>>> -    if (environ == env) {
>>>> +    char        *p, **penv;
>>>> +    size_t       size;
>>>> +    ngx_uint_t   i, n;
>>>> 
>>>> -        /*
>>>> -         * if the environment is still used, as it happens on exit,
>>>> -         * the only option is to leak it
>>>> -         */
>>>> -
>>>> +    if (environ != env) {
>>>> +        ngx_free(env);
>>>>        return;
>>>>    }
>>> 
>>> Note: if we assume that the environment can be arbitrary modified, 
>>> I don't think it's enough.  For example, a setenv() call can 
>>> replace environ with something different, so this code will free 
>>> env and exit, but pool-allocated variables will be still 
>>> referenced by the new environ.
>>> 
>>> This is probably out of the scope of this patch though.
>> 
>> It could be solved by always checking for pool-allocated variables,
>> irrespective of whether environ was changed after setenv(),
>> using that as an additional condition to free environment.
>> 
>>> 
>>>> 
>>>> -    ngx_free(env);
>>>> +    /*
>>>> +     * if the environment is still used, as it happens on exit,
>>>> +     * the only option is to leak it
>>>> +     */
>>>> +
>>>> +    for (n = 0; env[n]; n++) { }
>>>> +
>>>> +    for (i = 0; env[i]; i++) {
>>>> +
>>>> +        for (penv = ngx_os_environ; *penv; penv++) {
>>>> +            if (ngx_strcmp(*penv, env[i]) == 0) {
>>> 
>>> I don't think it's going to work.  If I'm reading this correctly, 
>>> with the initial environment "FOO=bar" and "env FOO=bar;" in the 
>>> configuration this will assume the string is from ngx_os_environ, 
>>> but it will be from ccf->env instead.
>>> 
>> 
>> Thanks for catching that.
>> So, we have to compare against ccf->env, instead.
>> 
>>>> +                break;
>>>> +            }
>>>> +        }
>>>> +
>>>> +        if (*penv) {
>>>> +            continue;
>>>> +        }
>>>> +
>>>> +        /* substitute pool allocated variables */
>>>> +
>>>> +        size = ngx_strlen(env[i]) + 1;
>>>> +        p = malloc(size);
>>> 
>>> There should be no direct malloc() calls except in low-level code 
>>> where logging is not available.  I don't think this particular 
>>> place counts, so ngx_alloc() should be used instead.
>> 
>> I used to avoid using cycle in cleanup, thus direct malloc() calls.
>> Anyway, it will be needed in order to access configuration.
>> 
>> Below is an updated patch (commit log unchanged).
>> 
>> # HG changeset patch
>> # User Sergey Kandaurov <pluknet at nginx.com>
>> # Date 1689267366 -14400
>> #      Thu Jul 13 20:56:06 2023 +0400
>> # Node ID 80a60d9707ed4b80402335d3d31a978f418f8056
>> # Parent  f91dc350be9f4a6bf1379a32a210afece7b0d75e
>> Core: fixed environment on exit better.
>> 
>> The fix in 6822:c045b4926b2c to replace environment allocations from a pool
>> was incomplete, such that it left pool owned allocations for variables set
>> with values using the "env" directive.  Another consumer of this interface
>> is the QUIC BPF helper that uses the environment variable NGINX_BPF_MAPS.
>> The fix is to re-allocate variables with malloc, to avoid use after free.
>> If memory allocation failed, the only option is to expel from environment.
>> 
>> The observed symptoms are similar to described in 6822:c045b4926b2c, that
>> is a segmentation fault on worker process exit from atexit() handler seen
>> with 3rd party modules or if nginx was built with lcov profiling.
>> 
>> diff --git a/src/core/nginx.c b/src/core/nginx.c
>> --- a/src/core/nginx.c
>> +++ b/src/core/nginx.c
>> @@ -592,7 +592,7 @@ tz_found:
>>         }
>> 
>>         cln->handler = ngx_cleanup_environment;
>> -        cln->data = env;
>> +        cln->data = cycle;
>>     }
>> 
>>     n = 0;
>> @@ -629,19 +629,57 @@ tz_found:
>> static void
>> ngx_cleanup_environment(void *data)
>> {
>> -    char  **env = data;
>> -
>> -    if (environ == env) {
>> +    ngx_cycle_t  *cycle = data;
>> 
>> -        /*
>> -         * if the environment is still used, as it happens on exit,
>> -         * the only option is to leak it
>> -         */
>> +    char             *p, **env;
>> +    size_t            size;
>> +    ngx_str_t        *var;
>> +    ngx_uint_t        i, n;
>> +    ngx_core_conf_t  *ccf;
>> 
>> +    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
>> +    env = ccf->environment;
>> +
>> +    if (environ != env) {
>> +        ngx_free(env);
>>         return;
>>     }
>> 
>> -    ngx_free(env);
>> +    /*
>> +     * if the environment is still used, as it happens on exit,
>> +     * the only option is to leak it
>> +     */
>> +
>> +    for (n = 0; env[n]; n++) { /* void */ }
>> +
>> +    while (*env) {
>> +        var = ccf->env.elts;
>> +
>> +        for (i = 0; i < ccf->env.nelts; i++) {
>> +            if (var[i].data == (u_char *) *env) {
>> +                break;
>> +            }
>> +        }
>> +
>> +        if (i == ccf->env.nelts) {
>> +            env++;
>> +            continue;
>> +        }
>> +
>> +        /* substitute pool allocated variables */
>> +
>> +        size = ngx_strlen(*env) + 1;
>> +        p = ngx_alloc(size, cycle->log);
>> +
>> +        if (p) {
>> +            ngx_memcpy(p, *env, size);
>> +            *env++ = p;
>> +
>> +        } else {
>> +            ngx_memmove(env, env + 1, sizeof(char *) * (n - (env - environ)));
>> +            n--;
>> +        }
>> +    }
>> }
>> 
>> 
>> 
>>> 
>>>> +
>>>> +        if (p) {
>>>> +            ngx_memcpy(p, env[i], size);
>>>> +            env[i] = p;
>>>> +
>>>> +        } else {
>>>> +            ngx_memmove(&env[i], &env[i + 1], sizeof(char *) * (n - i));
>>>> +            i--; n--;
>>>> +        }
>>>> +    }
>>>> }
>>> 
>>> Overall, it looks like a better idea might be to simply add all 
>>> the variables added to the environment to the environ allocation 
>>> in ngx_set_environment().
>>> 
>>> In ngx_set_environment() we explicitly know if the particular 
>>> variable is from OS environment or from ccf->environ, so there 
>>> will be no need to guess.  Also, the appropriate log is readily 
>>> available (and already used in the existing ngx_alloc() call), and 
>>> the code to cleanup things is already there and will work without 
>>> additional modifications.
>> 
>> I tried to avoid allocations in ngx_set_environment(),
>> as it can be called from the master process and harm
>> hypothetically by accumulating leaked allocations.
> 
> The ngx_set_environment() function installs a cleanup handler, 
> which is to be called on cycle pool destruction, and therefore 
> there can't be a leak unless the cleanup handler will decide it 
> needs to leak the allocation.
> 
> And, if the cleanup handler will decide to leak it, there should 
> be no difference with re-allocating things in the cleanup handler 
> (except the fact that allocations in cleanup handler are risky, 
> see below).

Let's take a look at the master/worker fork model, for clarity,
where allocations are decided to be leaked in the cleanup handler,
that is, in ngx_cleanup_environment().

In this model, cleanup handlers are registered on the master process
startup and on each reconfigure, and called when the master process
is exited, as part of destroying the pool in ngx_master_process_exit(),
and on each reconfigure, where old cycle pool is destroyed at the
end of successful ngx_init_cycle(), correspondingly.
Additionally, cleanup handlers are called on worker process exit
in ngx_worker_process_exit(), due to the fork model.

The difference is that if such allocations are postponed to the cleanup
handler in worker processes and to be leaked, this won't affect master
process memory, while worker processes are about to exit anyway.
In that sense, allocations should be guarded additionally with
ngx_process == NGX_PROCESS_WORKER.  Unlike that, and what you propose,
allocating things in the master process means they can be leaked there
as well, and such leakages can be accumulated with each reconfigure.

There is already an allocation of n+1 environment pointers, though,
in ngx_set_environment().
In this regard, it seems natural to continue allocations there, as
you propose, and address potential leakages separately, if at all.

Additionally, to make it clear what happens on reconfigure, the
master process appears to replace the environment before calling
cleanup handlers attributed to the old cycle, such that the proposed
ngx_cleanup_environment_variable() should be able to free memory
(that's not the case for process exit, but see above).
So leaking memory for environment variables seems to continue
to be rather hypothetical.

> 
> Rather, I would prefer to avoid allocations in 
> ngx_cleanup_environment(), since on failure this will corrupt the 
> environment with unpredictable results, and there are no options 
> to avoid this.

It is handled by removing such variables from environment
if allocation has failed.  This is rather odd behaviour though.
In general, allocations in cleanup look awkward and best to avoid.

>  Allocating everything in ngx_set_environment() 
> seems to be much safer approach.
> 
> I've tried a few variants, and the most simple one seems to 
> allocate variables one-by-one with appropriate cleanup handlers in 
> ngx_set_environment().  (Doing the same in ngx_set_env() is 
> slightly easier, but doesn't play well with QUIC.)  This approach 
> addresses all possible environment modifications and also looks 
> perfectly in line with the existing code.
> 
> Here is the patch:
> 
> # HG changeset patch
> # User Maxim Dounin <mdounin at mdounin.ru>
> # Date 1689637083 -10800
> #      Tue Jul 18 02:38:03 2023 +0300
> # Node ID dedb235210f816afc028f60a525009fc5dabd8bf
> # Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
> Core: fixed environment variables on exit.
> 
> Similarly to 6822:c045b4926b2c, environment variables introduced with
> the "env" directive (and "NGINX_BPF_MAPS" added by QUIC) are now allocated
> via ngx_alloc(), and explicitly freed by a cleanup handler if no longer used.
> 
> In collaboration with Sergey Kandaurov.
> 
> diff --git a/src/core/nginx.c b/src/core/nginx.c
> --- a/src/core/nginx.c
> +++ b/src/core/nginx.c
> @@ -13,6 +13,7 @@
> static void ngx_show_version_info(void);
> static ngx_int_t ngx_add_inherited_sockets(ngx_cycle_t *cycle);
> static void ngx_cleanup_environment(void *data);
> +static void ngx_cleanup_environment_variable(void *data);
> static ngx_int_t ngx_get_options(int argc, char *const *argv);
> static ngx_int_t ngx_process_options(ngx_cycle_t *cycle);
> static ngx_int_t ngx_save_argv(ngx_cycle_t *cycle, int argc, char *const *argv);
> @@ -518,7 +519,8 @@ ngx_add_inherited_sockets(ngx_cycle_t *c
> char **
> ngx_set_environment(ngx_cycle_t *cycle, ngx_uint_t *last)
> {
> -    char                **p, **env;
> +    char                **p, **env, *str;
> +    size_t                len;
>     ngx_str_t            *var;
>     ngx_uint_t            i, n;
>     ngx_core_conf_t      *ccf;
> @@ -600,7 +602,31 @@ tz_found:
>     for (i = 0; i < ccf->env.nelts; i++) {
> 
>         if (var[i].data[var[i].len] == '=') {
> -            env[n++] = (char *) var[i].data;
> +
> +            if (last) {
> +                env[n++] = (char *) var[i].data;
> +                continue;
> +            }
> +
> +            cln = ngx_pool_cleanup_add(cycle->pool, 0);
> +            if (cln == NULL) {
> +                return NULL;
> +            }
> +
> +            len = ngx_strlen(var[i].data) + 1;
> +
> +            str = ngx_alloc(len, cycle->log);
> +            if (str == NULL) {
> +                return NULL;
> +            }
> +
> +            ngx_memcpy(str, var[i].data, len);
> +
> +            cln->handler = ngx_cleanup_environment_variable;
> +            cln->data = str;
> +
> +            env[n++] = str;
> +
>             continue;
>         }
> 
> @@ -645,6 +671,29 @@ ngx_cleanup_environment(void *data)
> }
> 
> 
> +static void
> +ngx_cleanup_environment_variable(void *data)
> +{
> +    char  *var = data;
> +
> +    char  **p;
> +
> +    for (p = environ; *p; p++) {
> +
> +        /*
> +         * if an environment variable is still used, as it happens on exit,
> +         * the only option is to leak it
> +         */
> +
> +        if (*p == var) {
> +            return;
> +        }
> +    }
> +
> +    ngx_free(var);
> +}
> +
> +
> ngx_pid_t
> ngx_exec_new_binary(ngx_cycle_t *cycle, char *const *argv)
> {
> 

That said, I think your change is good.

-- 
Sergey Kandaurov

From mdounin at mdounin.ru  Wed Jul 19 02:31:18 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Wed, 19 Jul 2023 05:31:18 +0300
Subject: [PATCH] Core: fixed environment on exit better
In-Reply-To: <BEA3FB1D-0140-4FC7-A0F5-2B2EDA5C510F@nginx.com>
References: <34a8e1a4161542896c11.1687535747@enoparse.local>
 <ZJZPtrxW0uDtw/57@mdounin.ru>
 <74BC1CA2-9522-43AA-9EFD-24581BF6F3FB@nginx.com>
 <ZLXaeGERpd5BcoQz@mdounin.ru>
 <BEA3FB1D-0140-4FC7-A0F5-2B2EDA5C510F@nginx.com>
Message-ID: <ZLdK9jXjOVJwysOo@mdounin.ru>

Hello!

On Tue, Jul 18, 2023 at 09:51:49PM +0400, Sergey Kandaurov wrote:

> > On 18 Jul 2023, at 04:19, Maxim Dounin <mdounin at mdounin.ru> wrote:
> > 
> > Hello!
> > 
> > On Thu, Jul 13, 2023 at 08:58:44PM +0400, Sergey Kandaurov wrote:
> > 
> >>> On 24 Jun 2023, at 06:06, Maxim Dounin <mdounin at mdounin.ru> wrote:
> >>> 
> >>> Hello!
> >>> 
> >>> On Fri, Jun 23, 2023 at 07:55:47PM +0400, Sergey Kandaurov wrote:
> >>> 
> >>>> # HG changeset patch
> >>>> # User Sergey Kandaurov <pluknet at nginx.com>
> >>>> # Date 1687535739 -14400
> >>>> #      Fri Jun 23 19:55:39 2023 +0400
> >>>> # Node ID 34a8e1a4161542896c11c4a5c60d6a6fe1931e3d
> >>>> # Parent  c681f4906d838ac45b517fada4c4274ade03fd3c
> >>>> Core: fixed environment on exit better.
> >>>> 
> >>>> The fix in 6822:c045b4926b2c to replace environment allocations from a pool
> >>>> was incomplete, such that it left pool owned allocations for variables set
> >>>> with values using the "env" directive.  Another consumer of this interface
> >>>> is the QUIC BPF helper that uses the environment variable NGINX_BPF_MAPS.
> >>>> The fix is to re-allocate variables with malloc, to avoid use after free.
> >>>> If memory allocation failed, the only option is to expel from environment.
> >>>> 
> >>>> The observed symptoms are similar to described in 6822:c045b4926b2c, that
> >>>> is a segmentation fault on worker process exit from atexit() handler seen
> >>>> with 3rd party modules or if nginx was built with lcov profiling.
> >>>> 
> >>>> diff --git a/src/core/nginx.c b/src/core/nginx.c
> >>>> --- a/src/core/nginx.c
> >>>> +++ b/src/core/nginx.c
> >>>> @@ -631,17 +631,48 @@ ngx_cleanup_environment(void *data)
> >>>> {
> >>>>    char  **env = data;
> >>>> 
> >>>> -    if (environ == env) {
> >>>> +    char        *p, **penv;
> >>>> +    size_t       size;
> >>>> +    ngx_uint_t   i, n;
> >>>> 
> >>>> -        /*
> >>>> -         * if the environment is still used, as it happens on exit,
> >>>> -         * the only option is to leak it
> >>>> -         */
> >>>> -
> >>>> +    if (environ != env) {
> >>>> +        ngx_free(env);
> >>>>        return;
> >>>>    }
> >>> 
> >>> Note: if we assume that the environment can be arbitrary modified, 
> >>> I don't think it's enough.  For example, a setenv() call can 
> >>> replace environ with something different, so this code will free 
> >>> env and exit, but pool-allocated variables will be still 
> >>> referenced by the new environ.
> >>> 
> >>> This is probably out of the scope of this patch though.
> >> 
> >> It could be solved by always checking for pool-allocated variables,
> >> irrespective of whether environ was changed after setenv(),
> >> using that as an additional condition to free environment.
> >> 
> >>> 
> >>>> 
> >>>> -    ngx_free(env);
> >>>> +    /*
> >>>> +     * if the environment is still used, as it happens on exit,
> >>>> +     * the only option is to leak it
> >>>> +     */
> >>>> +
> >>>> +    for (n = 0; env[n]; n++) { }
> >>>> +
> >>>> +    for (i = 0; env[i]; i++) {
> >>>> +
> >>>> +        for (penv = ngx_os_environ; *penv; penv++) {
> >>>> +            if (ngx_strcmp(*penv, env[i]) == 0) {
> >>> 
> >>> I don't think it's going to work.  If I'm reading this correctly, 
> >>> with the initial environment "FOO=bar" and "env FOO=bar;" in the 
> >>> configuration this will assume the string is from ngx_os_environ, 
> >>> but it will be from ccf->env instead.
> >>> 
> >> 
> >> Thanks for catching that.
> >> So, we have to compare against ccf->env, instead.
> >> 
> >>>> +                break;
> >>>> +            }
> >>>> +        }
> >>>> +
> >>>> +        if (*penv) {
> >>>> +            continue;
> >>>> +        }
> >>>> +
> >>>> +        /* substitute pool allocated variables */
> >>>> +
> >>>> +        size = ngx_strlen(env[i]) + 1;
> >>>> +        p = malloc(size);
> >>> 
> >>> There should be no direct malloc() calls except in low-level code 
> >>> where logging is not available.  I don't think this particular 
> >>> place counts, so ngx_alloc() should be used instead.
> >> 
> >> I used to avoid using cycle in cleanup, thus direct malloc() calls.
> >> Anyway, it will be needed in order to access configuration.
> >> 
> >> Below is an updated patch (commit log unchanged).
> >> 
> >> # HG changeset patch
> >> # User Sergey Kandaurov <pluknet at nginx.com>
> >> # Date 1689267366 -14400
> >> #      Thu Jul 13 20:56:06 2023 +0400
> >> # Node ID 80a60d9707ed4b80402335d3d31a978f418f8056
> >> # Parent  f91dc350be9f4a6bf1379a32a210afece7b0d75e
> >> Core: fixed environment on exit better.
> >> 
> >> The fix in 6822:c045b4926b2c to replace environment allocations from a pool
> >> was incomplete, such that it left pool owned allocations for variables set
> >> with values using the "env" directive.  Another consumer of this interface
> >> is the QUIC BPF helper that uses the environment variable NGINX_BPF_MAPS.
> >> The fix is to re-allocate variables with malloc, to avoid use after free.
> >> If memory allocation failed, the only option is to expel from environment.
> >> 
> >> The observed symptoms are similar to described in 6822:c045b4926b2c, that
> >> is a segmentation fault on worker process exit from atexit() handler seen
> >> with 3rd party modules or if nginx was built with lcov profiling.
> >> 
> >> diff --git a/src/core/nginx.c b/src/core/nginx.c
> >> --- a/src/core/nginx.c
> >> +++ b/src/core/nginx.c
> >> @@ -592,7 +592,7 @@ tz_found:
> >>         }
> >> 
> >>         cln->handler = ngx_cleanup_environment;
> >> -        cln->data = env;
> >> +        cln->data = cycle;
> >>     }
> >> 
> >>     n = 0;
> >> @@ -629,19 +629,57 @@ tz_found:
> >> static void
> >> ngx_cleanup_environment(void *data)
> >> {
> >> -    char  **env = data;
> >> -
> >> -    if (environ == env) {
> >> +    ngx_cycle_t  *cycle = data;
> >> 
> >> -        /*
> >> -         * if the environment is still used, as it happens on exit,
> >> -         * the only option is to leak it
> >> -         */
> >> +    char             *p, **env;
> >> +    size_t            size;
> >> +    ngx_str_t        *var;
> >> +    ngx_uint_t        i, n;
> >> +    ngx_core_conf_t  *ccf;
> >> 
> >> +    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
> >> +    env = ccf->environment;
> >> +
> >> +    if (environ != env) {
> >> +        ngx_free(env);
> >>         return;
> >>     }
> >> 
> >> -    ngx_free(env);
> >> +    /*
> >> +     * if the environment is still used, as it happens on exit,
> >> +     * the only option is to leak it
> >> +     */
> >> +
> >> +    for (n = 0; env[n]; n++) { /* void */ }
> >> +
> >> +    while (*env) {
> >> +        var = ccf->env.elts;
> >> +
> >> +        for (i = 0; i < ccf->env.nelts; i++) {
> >> +            if (var[i].data == (u_char *) *env) {
> >> +                break;
> >> +            }
> >> +        }
> >> +
> >> +        if (i == ccf->env.nelts) {
> >> +            env++;
> >> +            continue;
> >> +        }
> >> +
> >> +        /* substitute pool allocated variables */
> >> +
> >> +        size = ngx_strlen(*env) + 1;
> >> +        p = ngx_alloc(size, cycle->log);
> >> +
> >> +        if (p) {
> >> +            ngx_memcpy(p, *env, size);
> >> +            *env++ = p;
> >> +
> >> +        } else {
> >> +            ngx_memmove(env, env + 1, sizeof(char *) * (n - (env - environ)));
> >> +            n--;
> >> +        }
> >> +    }
> >> }
> >> 
> >> 
> >> 
> >>> 
> >>>> +
> >>>> +        if (p) {
> >>>> +            ngx_memcpy(p, env[i], size);
> >>>> +            env[i] = p;
> >>>> +
> >>>> +        } else {
> >>>> +            ngx_memmove(&env[i], &env[i + 1], sizeof(char *) * (n - i));
> >>>> +            i--; n--;
> >>>> +        }
> >>>> +    }
> >>>> }
> >>> 
> >>> Overall, it looks like a better idea might be to simply add all 
> >>> the variables added to the environment to the environ allocation 
> >>> in ngx_set_environment().
> >>> 
> >>> In ngx_set_environment() we explicitly know if the particular 
> >>> variable is from OS environment or from ccf->environ, so there 
> >>> will be no need to guess.  Also, the appropriate log is readily 
> >>> available (and already used in the existing ngx_alloc() call), and 
> >>> the code to cleanup things is already there and will work without 
> >>> additional modifications.
> >> 
> >> I tried to avoid allocations in ngx_set_environment(),
> >> as it can be called from the master process and harm
> >> hypothetically by accumulating leaked allocations.
> > 
> > The ngx_set_environment() function installs a cleanup handler, 
> > which is to be called on cycle pool destruction, and therefore 
> > there can't be a leak unless the cleanup handler will decide it 
> > needs to leak the allocation.
> > 
> > And, if the cleanup handler will decide to leak it, there should 
> > be no difference with re-allocating things in the cleanup handler 
> > (except the fact that allocations in cleanup handler are risky, 
> > see below).
> 
> Let's take a look at the master/worker fork model, for clarity,
> where allocations are decided to be leaked in the cleanup handler,
> that is, in ngx_cleanup_environment().
> 
> In this model, cleanup handlers are registered on the master process
> startup and on each reconfigure, and called when the master process
> is exited, as part of destroying the pool in ngx_master_process_exit(),
> and on each reconfigure, where old cycle pool is destroyed at the
> end of successful ngx_init_cycle(), correspondingly.
> Additionally, cleanup handlers are called on worker process exit
> in ngx_worker_process_exit(), due to the fork model.
> 
> The difference is that if such allocations are postponed to the cleanup
> handler in worker processes and to be leaked, this won't affect master
> process memory, while worker processes are about to exit anyway.
> In that sense, allocations should be guarded additionally with
> ngx_process == NGX_PROCESS_WORKER.  Unlike that, and what you propose,
> allocating things in the master process means they can be leaked there
> as well, and such leakages can be accumulated with each reconfigure.

What happens in worker processes is indeed not really interesting, 
as environment there is expected to leak, and it will do so.

But in the master process you also have to preserve environment on 
exit as long as it was set there (notably, with the embedded perl 
module compiled in).  And you decide to leak or not based on the 
environ state in the cleanup handler, so there is no difference in 
whether the memory is allocated in ngx_set_environment() or in the 
cleanup handler.

> There is already an allocation of n+1 environment pointers, though,
> in ngx_set_environment().
> In this regard, it seems natural to continue allocations there, as
> you propose, and address potential leakages separately, if at all.
> 
> Additionally, to make it clear what happens on reconfigure, the
> master process appears to replace the environment before calling
> cleanup handlers attributed to the old cycle, such that the proposed
> ngx_cleanup_environment_variable() should be able to free memory
> (that's not the case for process exit, but see above).
> So leaking memory for environment variables seems to continue
> to be rather hypothetical.

Well, I tend to think that I actually know how to trigger a leak: 
if we compile perl module dynamically, and load it, it will call 
ngx_set_environment() during configuration parsing.  And if we'll 
unload it on the next configuration reload, environ won't be 
overwritten, so cleanup handler will leak it.  Then it can be 
loaded again, and so on.  (Nitpicking: on FreeBSD this also 
requires TZ to be set, since otherwise ngx_timezone_update() will 
update environ.)

This behaviour is, however, irrelevant to the approach taken, and 
will manifest itself regardless of the place where actual 
allocation happens.  (And, of course, this behaviour is already 
present with the current code.)

Not sure if fixing this worth the effort though.

> > 
> > Rather, I would prefer to avoid allocations in 
> > ngx_cleanup_environment(), since on failure this will corrupt the 
> > environment with unpredictable results, and there are no options 
> > to avoid this.
> 
> It is handled by removing such variables from environment
> if allocation has failed.

Sure, it is handled in your code.  Still, this handling will 
result in corrupted environment, without the variable in question, 
and results are unpredictable.

> This is rather odd behaviour though.
> In general, allocations in cleanup look awkward and best to avoid.

Exactly.

> 
> >  Allocating everything in ngx_set_environment() 
> > seems to be much safer approach.
> > 
> > I've tried a few variants, and the most simple one seems to 
> > allocate variables one-by-one with appropriate cleanup handlers in 
> > ngx_set_environment().  (Doing the same in ngx_set_env() is 
> > slightly easier, but doesn't play well with QUIC.)  This approach 
> > addresses all possible environment modifications and also looks 
> > perfectly in line with the existing code.
> > 
> > Here is the patch:
> > 
> > # HG changeset patch
> > # User Maxim Dounin <mdounin at mdounin.ru>
> > # Date 1689637083 -10800
> > #      Tue Jul 18 02:38:03 2023 +0300
> > # Node ID dedb235210f816afc028f60a525009fc5dabd8bf
> > # Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
> > Core: fixed environment variables on exit.
> > 
> > Similarly to 6822:c045b4926b2c, environment variables introduced with
> > the "env" directive (and "NGINX_BPF_MAPS" added by QUIC) are now allocated
> > via ngx_alloc(), and explicitly freed by a cleanup handler if no longer used.
> > 
> > In collaboration with Sergey Kandaurov.
> > 
> > diff --git a/src/core/nginx.c b/src/core/nginx.c
> > --- a/src/core/nginx.c
> > +++ b/src/core/nginx.c
> > @@ -13,6 +13,7 @@
> > static void ngx_show_version_info(void);
> > static ngx_int_t ngx_add_inherited_sockets(ngx_cycle_t *cycle);
> > static void ngx_cleanup_environment(void *data);
> > +static void ngx_cleanup_environment_variable(void *data);
> > static ngx_int_t ngx_get_options(int argc, char *const *argv);
> > static ngx_int_t ngx_process_options(ngx_cycle_t *cycle);
> > static ngx_int_t ngx_save_argv(ngx_cycle_t *cycle, int argc, char *const *argv);
> > @@ -518,7 +519,8 @@ ngx_add_inherited_sockets(ngx_cycle_t *c
> > char **
> > ngx_set_environment(ngx_cycle_t *cycle, ngx_uint_t *last)
> > {
> > -    char                **p, **env;
> > +    char                **p, **env, *str;
> > +    size_t                len;
> >     ngx_str_t            *var;
> >     ngx_uint_t            i, n;
> >     ngx_core_conf_t      *ccf;
> > @@ -600,7 +602,31 @@ tz_found:
> >     for (i = 0; i < ccf->env.nelts; i++) {
> > 
> >         if (var[i].data[var[i].len] == '=') {
> > -            env[n++] = (char *) var[i].data;
> > +
> > +            if (last) {
> > +                env[n++] = (char *) var[i].data;
> > +                continue;
> > +            }
> > +
> > +            cln = ngx_pool_cleanup_add(cycle->pool, 0);
> > +            if (cln == NULL) {
> > +                return NULL;
> > +            }
> > +
> > +            len = ngx_strlen(var[i].data) + 1;
> > +
> > +            str = ngx_alloc(len, cycle->log);
> > +            if (str == NULL) {
> > +                return NULL;
> > +            }
> > +
> > +            ngx_memcpy(str, var[i].data, len);
> > +
> > +            cln->handler = ngx_cleanup_environment_variable;
> > +            cln->data = str;
> > +
> > +            env[n++] = str;
> > +
> >             continue;
> >         }
> > 
> > @@ -645,6 +671,29 @@ ngx_cleanup_environment(void *data)
> > }
> > 
> > 
> > +static void
> > +ngx_cleanup_environment_variable(void *data)
> > +{
> > +    char  *var = data;
> > +
> > +    char  **p;
> > +
> > +    for (p = environ; *p; p++) {
> > +
> > +        /*
> > +         * if an environment variable is still used, as it happens on exit,
> > +         * the only option is to leak it
> > +         */
> > +
> > +        if (*p == var) {
> > +            return;
> > +        }
> > +    }
> > +
> > +    ngx_free(var);
> > +}
> > +
> > +
> > ngx_pid_t
> > ngx_exec_new_binary(ngx_cycle_t *cycle, char *const *argv)
> > {
> > 
> 
> That said, I think your change is good.

Pushed to http://mdounin.ru/hg/nginx, thanks for looking.

-- 
Maxim Dounin
http://mdounin.ru/

From pluknet at nginx.com  Wed Jul 19 08:07:29 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Wed, 19 Jul 2023 08:07:29 +0000
Subject: [nginx] Core: fixed environment variables on exit.
Message-ID: <hg.c209dc4eed17.1689754049.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/c209dc4eed17
branches:  
changeset: 9134:c209dc4eed17
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Wed Jul 19 05:09:23 2023 +0300
description:
Core: fixed environment variables on exit.

Similarly to 6822:c045b4926b2c, environment variables introduced with
the "env" directive (and "NGINX_BPF_MAPS" added by QUIC) are now allocated
via ngx_alloc(), and explicitly freed by a cleanup handler if no longer used.

In collaboration with Sergey Kandaurov.

diffstat:

 src/core/nginx.c |  53 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 files changed, 51 insertions(+), 2 deletions(-)

diffs (84 lines):

diff -r f91dc350be9f -r c209dc4eed17 src/core/nginx.c
--- a/src/core/nginx.c	Wed Jul 12 15:27:35 2023 +0400
+++ b/src/core/nginx.c	Wed Jul 19 05:09:23 2023 +0300
@@ -13,6 +13,7 @@
 static void ngx_show_version_info(void);
 static ngx_int_t ngx_add_inherited_sockets(ngx_cycle_t *cycle);
 static void ngx_cleanup_environment(void *data);
+static void ngx_cleanup_environment_variable(void *data);
 static ngx_int_t ngx_get_options(int argc, char *const *argv);
 static ngx_int_t ngx_process_options(ngx_cycle_t *cycle);
 static ngx_int_t ngx_save_argv(ngx_cycle_t *cycle, int argc, char *const *argv);
@@ -518,7 +519,8 @@ ngx_add_inherited_sockets(ngx_cycle_t *c
 char **
 ngx_set_environment(ngx_cycle_t *cycle, ngx_uint_t *last)
 {
-    char                **p, **env;
+    char                **p, **env, *str;
+    size_t                len;
     ngx_str_t            *var;
     ngx_uint_t            i, n;
     ngx_core_conf_t      *ccf;
@@ -600,7 +602,31 @@ tz_found:
     for (i = 0; i < ccf->env.nelts; i++) {
 
         if (var[i].data[var[i].len] == '=') {
-            env[n++] = (char *) var[i].data;
+
+            if (last) {
+                env[n++] = (char *) var[i].data;
+                continue;
+            }
+
+            cln = ngx_pool_cleanup_add(cycle->pool, 0);
+            if (cln == NULL) {
+                return NULL;
+            }
+
+            len = ngx_strlen(var[i].data) + 1;
+
+            str = ngx_alloc(len, cycle->log);
+            if (str == NULL) {
+                return NULL;
+            }
+
+            ngx_memcpy(str, var[i].data, len);
+
+            cln->handler = ngx_cleanup_environment_variable;
+            cln->data = str;
+
+            env[n++] = str;
+
             continue;
         }
 
@@ -645,6 +671,29 @@ ngx_cleanup_environment(void *data)
 }
 
 
+static void
+ngx_cleanup_environment_variable(void *data)
+{
+    char  *var = data;
+
+    char  **p;
+
+    for (p = environ; *p; p++) {
+
+        /*
+         * if an environment variable is still used, as it happens on exit,
+         * the only option is to leak it
+         */
+
+        if (*p == var) {
+            return;
+        }
+    }
+
+    ngx_free(var);
+}
+
+
 ngx_pid_t
 ngx_exec_new_binary(ngx_cycle_t *cycle, char *const *argv)
 {

From gmm at csdoc.com  Wed Jul 19 21:50:19 2023
From: gmm at csdoc.com (Gena Makhomed)
Date: Thu, 20 Jul 2023 00:50:19 +0300
Subject: [PATCH] Contrib: vim syntax, update core and 3rd party module
 directives.
Message-ID: <5f76c221-445a-ff97-f9b2-04430b0902d1@csdoc.com>

# HG changeset patch
# User Gena Makhomed <gmm at csdoc.com>
# Date 1689803114 -10800
#      Thu Jul 20 00:45:14 2023 +0300
# Node ID dee0e911380d18773dad06d45b845a124474f12a
# Parent  c209dc4eed1737261c97a044d23f5fb9afbd9d45
Contrib: vim syntax, update core and 3rd party module directives.

List of 3rd party modules github repositories are obtained from
https://github.com/freebsd/freebsd-ports/blob/main/www/nginx-devel/Makefile.extmod

diff -r c209dc4eed17 -r dee0e911380d contrib/vim/syntax/nginx.vim
--- a/contrib/vim/syntax/nginx.vim	Wed Jul 19 05:09:23 2023 +0300
+++ b/contrib/vim/syntax/nginx.vim	Thu Jul 20 00:45:14 2023 +0300
@@ -65,43 +65,44 @@
      \ contained
      \ nextgroup=@ngxListenParams skipwhite skipempty
  syn keyword ngxListenOptions contained
-    \ default_server ssl http2 proxy_protocol
+    \ default_server ssl quic proxy_protocol
      \ setfib fastopen backlog rcvbuf sndbuf accept_filter deferred bind
      \ ipv6only reuseport so_keepalive
      \ nextgroup=@ngxListenParams skipwhite skipempty
  syn keyword ngxListenOptionsDeprecated contained
-    \ spdy
+    \ http2
      \ nextgroup=@ngxListenParams skipwhite skipempty
  syn cluster ngxListenParams
      \ contains=ngxListenParam,ngxListenString,ngxListenComment
      \ add=ngxListenOptions,ngxListenOptionsDeprecated

-syn keyword ngxDirectiveBlock contained http
-syn keyword ngxDirectiveBlock contained stream
-syn keyword ngxDirectiveBlock contained mail
+syn keyword ngxDirectiveBlock contained charset_map
  syn keyword ngxDirectiveBlock contained events
-syn keyword ngxDirectiveBlock contained server
-syn keyword ngxDirectiveBlock contained types
+syn keyword ngxDirectiveBlock contained geo
+syn keyword ngxDirectiveBlock contained http
+syn keyword ngxDirectiveBlock contained if
+syn keyword ngxDirectiveBlock contained limit_except
  syn keyword ngxDirectiveBlock contained location
-syn keyword ngxDirectiveBlock contained upstream
-syn keyword ngxDirectiveBlock contained charset_map
-syn keyword ngxDirectiveBlock contained limit_except
-syn keyword ngxDirectiveBlock contained if
-syn keyword ngxDirectiveBlock contained geo
+syn keyword ngxDirectiveBlock contained mail
  syn keyword ngxDirectiveBlock contained map
+syn keyword ngxDirectiveBlock contained match
+syn keyword ngxDirectiveBlock contained otel_exporter
+syn keyword ngxDirectiveBlock contained server
  syn keyword ngxDirectiveBlock contained split_clients
-syn keyword ngxDirectiveBlock contained match
+syn keyword ngxDirectiveBlock contained stream
+syn keyword ngxDirectiveBlock contained types
+syn keyword ngxDirectiveBlock contained upstream

+syn keyword ngxDirectiveImportant contained fastcgi_pass
  syn keyword ngxDirectiveImportant contained include
-syn keyword ngxDirectiveImportant contained root
-syn keyword ngxDirectiveImportant contained server_name
  syn keyword ngxDirectiveImportant contained internal
+syn keyword ngxDirectiveImportant contained memcached_pass
  syn keyword ngxDirectiveImportant contained proxy_pass
-syn keyword ngxDirectiveImportant contained memcached_pass
-syn keyword ngxDirectiveImportant contained fastcgi_pass
+syn keyword ngxDirectiveImportant contained root
  syn keyword ngxDirectiveImportant contained scgi_pass
+syn keyword ngxDirectiveImportant contained server_name
+syn keyword ngxDirectiveImportant contained try_files
  syn keyword ngxDirectiveImportant contained uwsgi_pass
-syn keyword ngxDirectiveImportant contained try_files

  syn keyword ngxDirectiveControl contained break
  syn keyword ngxDirectiveControl contained return
@@ -111,25 +112,24 @@
  syn keyword ngxDirectiveError contained error_page
  syn keyword ngxDirectiveError contained post_action

-syn keyword ngxDirectiveDeprecated contained proxy_downstream_buffer
-syn keyword ngxDirectiveDeprecated contained proxy_upstream_buffer
-syn keyword ngxDirectiveDeprecated contained ssl
  syn keyword ngxDirectiveDeprecated contained http2_idle_timeout
  syn keyword ngxDirectiveDeprecated contained http2_max_field_size
  syn keyword ngxDirectiveDeprecated contained http2_max_header_size
  syn keyword ngxDirectiveDeprecated contained http2_max_requests
  syn keyword ngxDirectiveDeprecated contained http2_recv_timeout
+syn keyword ngxDirectiveDeprecated contained proxy_downstream_buffer
+syn keyword ngxDirectiveDeprecated contained proxy_upstream_buffer

  syn keyword ngxDirective contained absolute_redirect
+syn keyword ngxDirective contained acceptex_read
  syn keyword ngxDirective contained accept_mutex
  syn keyword ngxDirective contained accept_mutex_delay
-syn keyword ngxDirective contained acceptex_read
  syn keyword ngxDirective contained access_log
  syn keyword ngxDirective contained add_after_body
  syn keyword ngxDirective contained add_before_body
  syn keyword ngxDirective contained add_header
+syn keyword ngxDirective contained addition_types
  syn keyword ngxDirective contained add_trailer
-syn keyword ngxDirective contained addition_types
  syn keyword ngxDirective contained aio
  syn keyword ngxDirective contained aio_write
  syn keyword ngxDirective contained alias
@@ -196,9 +196,9 @@
  syn keyword ngxDirective contained f4f
  syn keyword ngxDirective contained f4f_buffer_size
  syn keyword ngxDirective contained fastcgi_bind
-syn keyword ngxDirective contained fastcgi_buffer_size
  syn keyword ngxDirective contained fastcgi_buffering
  syn keyword ngxDirective contained fastcgi_buffers
+syn keyword ngxDirective contained fastcgi_buffer_size
  syn keyword ngxDirective contained fastcgi_busy_buffers_size
  syn keyword ngxDirective contained fastcgi_cache
  syn keyword ngxDirective contained fastcgi_cache_background_update
@@ -303,6 +303,7 @@
  syn keyword ngxDirective contained hls_fragment
  syn keyword ngxDirective contained hls_mp4_buffer_size
  syn keyword ngxDirective contained hls_mp4_max_buffer_size
+syn keyword ngxDirective contained http2
  syn keyword ngxDirective contained http2_body_preread_size
  syn keyword ngxDirective contained http2_chunk_size
  syn keyword ngxDirective contained http2_max_concurrent_pushes
@@ -312,6 +313,10 @@
  syn keyword ngxDirective contained http2_push_preload
  syn keyword ngxDirective contained http2_recv_buffer_size
  syn keyword ngxDirective contained http2_streams_index_size
+syn keyword ngxDirective contained http3
+syn keyword ngxDirective contained http3_hq
+syn keyword ngxDirective contained http3_max_concurrent_streams
+syn keyword ngxDirective contained http3_stream_buffer_size
  syn keyword ngxDirective contained if_modified_since
  syn keyword ngxDirective contained ignore_invalid_headers
  syn keyword ngxDirective contained image_filter
@@ -325,6 +330,7 @@
  syn keyword ngxDirective contained imap_capabilities
  syn keyword ngxDirective contained imap_client_buffer
  syn keyword ngxDirective contained index
+syn keyword ngxDirective contained internal_redirect
  syn keyword ngxDirective contained iocp_threads
  syn keyword ngxDirective contained ip_hash
  syn keyword ngxDirective contained js_access
@@ -342,8 +348,10 @@
  syn keyword ngxDirective contained js_header_filter
  syn keyword ngxDirective contained js_import
  syn keyword ngxDirective contained js_path
+syn keyword ngxDirective contained js_preload_object
  syn keyword ngxDirective contained js_preread
  syn keyword ngxDirective contained js_set
+syn keyword ngxDirective contained js_shared_dict_zone
  syn keyword ngxDirective contained js_var
  syn keyword ngxDirective contained keepalive
  syn keyword ngxDirective contained keepalive_disable
@@ -404,6 +412,10 @@
  syn keyword ngxDirective contained mp4_limit_rate_after
  syn keyword ngxDirective contained mp4_max_buffer_size
  syn keyword ngxDirective contained mp4_start_key_frame
+syn keyword ngxDirective contained mqtt
+syn keyword ngxDirective contained mqtt_preread
+syn keyword ngxDirective contained mqtt_rewrite_buffer_size
+syn keyword ngxDirective contained mqtt_set_connect
  syn keyword ngxDirective contained msie_padding
  syn keyword ngxDirective contained msie_refresh
  syn keyword ngxDirective contained multi_accept
@@ -414,6 +426,11 @@
  syn keyword ngxDirective contained open_file_cache_min_uses
  syn keyword ngxDirective contained open_file_cache_valid
  syn keyword ngxDirective contained open_log_file_cache
+syn keyword ngxDirective contained otel_service_name
+syn keyword ngxDirective contained otel_span_attr
+syn keyword ngxDirective contained otel_span_name
+syn keyword ngxDirective contained otel_trace
+syn keyword ngxDirective contained otel_trace_context
  syn keyword ngxDirective contained output_buffers
  syn keyword ngxDirective contained override_charset
  syn keyword ngxDirective contained pcre_jit
@@ -434,9 +451,9 @@
  syn keyword ngxDirective contained proxy
  syn keyword ngxDirective contained proxy_bind
  syn keyword ngxDirective contained proxy_buffer
-syn keyword ngxDirective contained proxy_buffer_size
  syn keyword ngxDirective contained proxy_buffering
  syn keyword ngxDirective contained proxy_buffers
+syn keyword ngxDirective contained proxy_buffer_size
  syn keyword ngxDirective contained proxy_busy_buffers_size
  syn keyword ngxDirective contained proxy_cache
  syn keyword ngxDirective contained proxy_cache_background_update
@@ -514,6 +531,11 @@
  syn keyword ngxDirective contained proxy_timeout
  syn keyword ngxDirective contained proxy_upload_rate
  syn keyword ngxDirective contained queue
+syn keyword ngxDirective contained quic_active_connection_id_limit
+syn keyword ngxDirective contained quic_bpf
+syn keyword ngxDirective contained quic_gso
+syn keyword ngxDirective contained quic_host_key
+syn keyword ngxDirective contained quic_retry
  syn keyword ngxDirective contained random
  syn keyword ngxDirective contained random_index
  syn keyword ngxDirective contained read_ahead
@@ -529,9 +551,9 @@
  syn keyword ngxDirective contained rewrite_log
  syn keyword ngxDirective contained satisfy
  syn keyword ngxDirective contained scgi_bind
-syn keyword ngxDirective contained scgi_buffer_size
  syn keyword ngxDirective contained scgi_buffering
  syn keyword ngxDirective contained scgi_buffers
+syn keyword ngxDirective contained scgi_buffer_size
  syn keyword ngxDirective contained scgi_busy_buffers_size
  syn keyword ngxDirective contained scgi_cache
  syn keyword ngxDirective contained scgi_cache_background_update
@@ -575,10 +597,10 @@
  syn keyword ngxDirective contained secure_link
  syn keyword ngxDirective contained secure_link_md5
  syn keyword ngxDirective contained secure_link_secret
+syn keyword ngxDirective contained sendfile
+syn keyword ngxDirective contained sendfile_max_chunk
  syn keyword ngxDirective contained send_lowat
  syn keyword ngxDirective contained send_timeout
-syn keyword ngxDirective contained sendfile
-syn keyword ngxDirective contained sendfile_max_chunk
  syn keyword ngxDirective contained server_name_in_redirect
  syn keyword ngxDirective contained server_names_hash_bucket_size
  syn keyword ngxDirective contained server_names_hash_max_size
@@ -665,9 +687,9 @@
  syn keyword ngxDirective contained userid_path
  syn keyword ngxDirective contained userid_service
  syn keyword ngxDirective contained uwsgi_bind
-syn keyword ngxDirective contained uwsgi_buffer_size
  syn keyword ngxDirective contained uwsgi_buffering
  syn keyword ngxDirective contained uwsgi_buffers
+syn keyword ngxDirective contained uwsgi_buffer_size
  syn keyword ngxDirective contained uwsgi_busy_buffers_size
  syn keyword ngxDirective contained uwsgi_cache
  syn keyword ngxDirective contained uwsgi_cache_background_update
@@ -837,52 +859,6 @@
  " https://github.com/torden/ngx_cache_purge
  syn keyword ngxDirectiveThirdParty contained cache_purge_response_type

-" https://github.com/nginx-clojure/nginx-clojure
-syn keyword ngxDirectiveThirdParty contained access_handler_code
-syn keyword ngxDirectiveThirdParty contained access_handler_name
-syn keyword ngxDirectiveThirdParty contained access_handler_property
-syn keyword ngxDirectiveThirdParty contained access_handler_type
-syn keyword ngxDirectiveThirdParty contained always_read_body
-syn keyword ngxDirectiveThirdParty contained auto_upgrade_ws
-syn keyword ngxDirectiveThirdParty contained body_filter_code
-syn keyword ngxDirectiveThirdParty contained body_filter_name
-syn keyword ngxDirectiveThirdParty contained body_filter_property
-syn keyword ngxDirectiveThirdParty contained body_filter_type
-syn keyword ngxDirectiveThirdParty contained content_handler_code
-syn keyword ngxDirectiveThirdParty contained content_handler_name
-syn keyword ngxDirectiveThirdParty contained content_handler_property
-syn keyword ngxDirectiveThirdParty contained content_handler_type
-syn keyword ngxDirectiveThirdParty contained handler_code
-syn keyword ngxDirectiveThirdParty contained handler_name
-syn keyword ngxDirectiveThirdParty contained handler_type
-syn keyword ngxDirectiveThirdParty contained handlers_lazy_init
-syn keyword ngxDirectiveThirdParty contained header_filter_code
-syn keyword ngxDirectiveThirdParty contained header_filter_name
-syn keyword ngxDirectiveThirdParty contained header_filter_property
-syn keyword ngxDirectiveThirdParty contained header_filter_type
-syn keyword ngxDirectiveThirdParty contained jvm_classpath
-syn keyword ngxDirectiveThirdParty contained jvm_classpath_check
-syn keyword ngxDirectiveThirdParty contained jvm_exit_handler_code
-syn keyword ngxDirectiveThirdParty contained jvm_exit_handler_name
-syn keyword ngxDirectiveThirdParty contained jvm_handler_type
-syn keyword ngxDirectiveThirdParty contained jvm_init_handler_code
-syn keyword ngxDirectiveThirdParty contained jvm_init_handler_name
-syn keyword ngxDirectiveThirdParty contained jvm_options
-syn keyword ngxDirectiveThirdParty contained jvm_path
-syn keyword ngxDirectiveThirdParty contained jvm_var
-syn keyword ngxDirectiveThirdParty contained jvm_workers
-syn keyword ngxDirectiveThirdParty contained log_handler_code
-syn keyword ngxDirectiveThirdParty contained log_handler_name
-syn keyword ngxDirectiveThirdParty contained log_handler_property
-syn keyword ngxDirectiveThirdParty contained log_handler_type
-syn keyword ngxDirectiveThirdParty contained max_balanced_tcp_connections
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_code
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_name
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_property
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_type
-syn keyword ngxDirectiveThirdParty contained shared_map
-syn keyword ngxDirectiveThirdParty contained write_page_size
-
  " https://github.com/AirisX/nginx_cookie_flag_module
  syn keyword ngxDirectiveThirdParty contained set_cookie_flag

@@ -932,29 +908,6 @@
  syn keyword ngxDirectiveThirdParty contained dynamic_state_file
  syn keyword ngxDirectiveThirdParty contained dynamic_upstream

-" https://github.com/ZigzagAK/ngx_dynamic_healthcheck
-syn keyword ngxDirectiveThirdParty contained check
-syn keyword ngxDirectiveThirdParty contained check_disable_host
-syn keyword ngxDirectiveThirdParty contained check_exclude_host
-syn keyword ngxDirectiveThirdParty contained check_persistent
-syn keyword ngxDirectiveThirdParty contained check_request_body
-syn keyword ngxDirectiveThirdParty contained check_request_headers
-syn keyword ngxDirectiveThirdParty contained check_request_uri
-syn keyword ngxDirectiveThirdParty contained check_response_body
-syn keyword ngxDirectiveThirdParty contained check_response_codes
-syn keyword ngxDirectiveThirdParty contained healthcheck
-syn keyword ngxDirectiveThirdParty contained healthcheck_buffer_size
-syn keyword ngxDirectiveThirdParty contained healthcheck_disable_host
-syn keyword ngxDirectiveThirdParty contained healthcheck_get
-syn keyword ngxDirectiveThirdParty contained healthcheck_persistent
-syn keyword ngxDirectiveThirdParty contained healthcheck_request_body
-syn keyword ngxDirectiveThirdParty contained healthcheck_request_headers
-syn keyword ngxDirectiveThirdParty contained healthcheck_request_uri
-syn keyword ngxDirectiveThirdParty contained healthcheck_response_body
-syn keyword ngxDirectiveThirdParty contained healthcheck_response_codes
-syn keyword ngxDirectiveThirdParty contained healthcheck_status
-syn keyword ngxDirectiveThirdParty contained healthcheck_update
-
  " https://github.com/openresty/encrypted-session-nginx-module
  syn keyword ngxDirectiveThirdParty contained encrypted_session_expires
  syn keyword ngxDirectiveThirdParty contained encrypted_session_iv
@@ -1004,6 +957,7 @@
  syn keyword ngxDirectiveThirdParty contained auth_gss_realm
  syn keyword ngxDirectiveThirdParty contained auth_gss_service_ccache
  syn keyword ngxDirectiveThirdParty contained auth_gss_service_name
+syn keyword ngxDirectiveThirdParty contained auth_gss_zone_name

  " https://github.com/kvspb/nginx-auth-ldap
  syn keyword ngxDirectiveThirdParty contained auth_ldap
@@ -1033,6 +987,7 @@

  " https://github.com/aperezdc/ngx-fancyindex
  syn keyword ngxDirectiveThirdParty contained fancyindex
+syn keyword ngxDirectiveThirdParty contained fancyindex_case_sensitive
  syn keyword ngxDirectiveThirdParty contained fancyindex_css_href
  syn keyword ngxDirectiveThirdParty contained fancyindex_default_sort
  syn keyword ngxDirectiveThirdParty contained fancyindex_directories_first
@@ -1082,8 +1037,8 @@
  syn keyword ngxDirectiveThirdParty contained 
nchan_benchmark_subscriber_distribution
  syn keyword ngxDirectiveThirdParty contained 
nchan_benchmark_subscribers_per_channel
  syn keyword ngxDirectiveThirdParty contained nchan_benchmark_time
+syn keyword ngxDirectiveThirdParty contained 
nchan_channel_events_channel_id
  syn keyword ngxDirectiveThirdParty contained nchan_channel_event_string
-syn keyword ngxDirectiveThirdParty contained 
nchan_channel_events_channel_id
  syn keyword ngxDirectiveThirdParty contained nchan_channel_group
  syn keyword ngxDirectiveThirdParty contained 
nchan_channel_group_accounting
  syn keyword ngxDirectiveThirdParty contained nchan_channel_id
@@ -1121,6 +1076,7 @@
  syn keyword ngxDirectiveThirdParty contained nchan_pubsub
  syn keyword ngxDirectiveThirdParty contained nchan_pubsub_channel_id
  syn keyword ngxDirectiveThirdParty contained nchan_pubsub_location
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_accurate_subscriber_count
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_cluster_check_interval
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_cluster_check_interval_backoff
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_cluster_check_interval_jitter
@@ -1138,6 +1094,11 @@
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_discovered_ip_range_blacklist
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_fakesub_timer_interval
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_cache_timeout
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_keepalive_backoff
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_keepalive_jitter
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_keepalive_max
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_keepalive_min
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_keepalive_safety_margin
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_load_scripts_unconditionally
  syn keyword ngxDirectiveThirdParty contained nchan_redis_namespace
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_node_connect_timeout
@@ -1173,6 +1134,9 @@
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_tls_trusted_certificate
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_tls_trusted_certificate_path
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_tls_verify_certificate
+syn keyword ngxDirectiveThirdParty contained nchan_redis_upstream_stats
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_upstream_stats_disconnected_timeout
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_upstream_stats_enabled
  syn keyword ngxDirectiveThirdParty contained nchan_redis_url
  syn keyword ngxDirectiveThirdParty contained nchan_redis_username
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_wait_after_connecting
@@ -1182,10 +1146,10 @@
  syn keyword ngxDirectiveThirdParty contained nchan_stub_status
  syn keyword ngxDirectiveThirdParty contained nchan_sub_channel_id
  syn keyword ngxDirectiveThirdParty contained 
nchan_subscribe_existing_channels_only
-syn keyword ngxDirectiveThirdParty contained nchan_subscribe_request
  syn keyword ngxDirectiveThirdParty contained nchan_subscriber
  syn keyword ngxDirectiveThirdParty contained nchan_subscriber_channel_id
  syn keyword ngxDirectiveThirdParty contained 
nchan_subscriber_compound_etag_message_id
+syn keyword ngxDirectiveThirdParty contained nchan_subscribe_request
  syn keyword ngxDirectiveThirdParty contained 
nchan_subscriber_first_message
  syn keyword ngxDirectiveThirdParty contained 
nchan_subscriber_http_raw_stream_separator
  syn keyword ngxDirectiveThirdParty contained nchan_subscriber_info
@@ -1214,6 +1178,7 @@
  syn keyword ngxDirectiveThirdParty contained push_subscriber_concurrency
  syn keyword ngxDirectiveThirdParty contained push_subscriber_timeout

+
  " https://github.com/wandenberg/nginx-push-stream-module
  syn keyword ngxDirectiveThirdParty contained 
push_stream_allow_connections_to_events_channel
  syn keyword ngxDirectiveThirdParty contained push_stream_allowed_origins
@@ -1323,6 +1288,7 @@
  syn keyword ngxDirectiveThirdParty contained upload_progress_template

  " https://github.com/yaoweibin/nginx_upstream_check_module
+syn keyword ngxDirectiveThirdParty contained check
  syn keyword ngxDirectiveThirdParty contained check_fastcgi_param
  syn keyword ngxDirectiveThirdParty contained check_http_expect_alive
  syn keyword ngxDirectiveThirdParty contained check_http_send
@@ -1335,6 +1301,7 @@
  syn keyword ngxDirectiveThirdParty contained upstream_fair_shm_size

  " https://github.com/ayty-adrianomartins/nginx-sticky-module-ng
+syn keyword ngxDirectiveThirdParty contained sticky_hide_cookie
  syn keyword ngxDirectiveThirdParty contained sticky_no_fallback

  " https://github.com/Novetta/nginx-video-thumbextractor-module
@@ -1421,6 +1388,8 @@
  syn keyword ngxDirectiveThirdParty contained lua_socket_read_timeout
  syn keyword ngxDirectiveThirdParty contained lua_socket_send_lowat
  syn keyword ngxDirectiveThirdParty contained lua_socket_send_timeout
+syn keyword ngxDirectiveThirdParty contained lua_ssl_certificate
+syn keyword ngxDirectiveThirdParty contained lua_ssl_certificate_key
  syn keyword ngxDirectiveThirdParty contained lua_ssl_ciphers
  syn keyword ngxDirectiveThirdParty contained lua_ssl_conf_command
  syn keyword ngxDirectiveThirdParty contained lua_ssl_crl
@@ -1449,6 +1418,7 @@
  syn keyword ngxDirectiveThirdParty contained 
ssl_session_store_by_lua_block
  syn keyword ngxDirectiveThirdParty contained ssl_session_store_by_lua_file

+
  " https://github.com/Taymindis/nginx-link-function
  syn keyword ngxDirectiveThirdParty contained ngx_link_func_add_prop
  syn keyword ngxDirectiveThirdParty contained ngx_link_func_add_req_header
@@ -1834,16 +1804,6 @@
  syn keyword ngxDirectiveThirdParty contained slowfs_cache_valid
  syn keyword ngxDirectiveThirdParty contained slowfs_temp_path

-" https://github.com/kawakibi/ngx_small_light
-syn keyword ngxDirectiveThirdParty contained small_light
-syn keyword ngxDirectiveThirdParty contained small_light_buffer
-syn keyword ngxDirectiveThirdParty contained small_light_getparam_mode
-syn keyword ngxDirectiveThirdParty contained small_light_imlib2_temp_dir
-syn keyword ngxDirectiveThirdParty contained small_light_material_dir
-syn keyword ngxDirectiveThirdParty contained small_light_pattern_define
-syn keyword ngxDirectiveThirdParty contained small_light_radius_max
-syn keyword ngxDirectiveThirdParty contained small_light_sigma_max
-
  " https://github.com/openresty/srcache-nginx-module
  syn keyword ngxDirectiveThirdParty contained srcache_buffer
  syn keyword ngxDirectiveThirdParty contained srcache_default_expire
@@ -1980,6 +1940,14 @@
  syn keyword ngxDirectiveThirdParty contained websockify_read_timeout
  syn keyword ngxDirectiveThirdParty contained websockify_send_timeout

+" https://github.com/vozlt/nginx-module-sts
+syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status
+syn keyword ngxDirectiveThirdParty contained 
stream_server_traffic_status_average_method
+syn keyword ngxDirectiveThirdParty contained 
stream_server_traffic_status_display
+syn keyword ngxDirectiveThirdParty contained 
stream_server_traffic_status_display_format
+syn keyword ngxDirectiveThirdParty contained 
stream_server_traffic_status_display_jsonp
+syn keyword ngxDirectiveThirdParty contained 
stream_server_traffic_status_zone
+
  " highlight

  hi def link ngxComment Comment

From pluknet at nginx.com  Thu Jul 20 11:56:54 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Thu, 20 Jul 2023 15:56:54 +0400
Subject: Is ngx_http_headers_out dead code?
In-Reply-To: <CAJuLd0j126x7BuLhwrwnUJNZcjpa9L6D_ACmAFekcaCuhsDYNQ@mail.gmail.com>
References: <CAJuLd0j126x7BuLhwrwnUJNZcjpa9L6D_ACmAFekcaCuhsDYNQ@mail.gmail.com>
Message-ID: <B059DB04-0D69-4D4D-A73A-BE7B78AA66C6@nginx.com>


> On 25 Jun 2023, at 11:25, 洪志道 <hongzhidao at gmail.com> wrote:
> 
> Hi,
> I wonder if the `ngx_http_headers_out` variable is being used? If yes, what is it for?
> 
> ```
> /* src/http/ngx_http_header_filter_module.c */
> 
>  ngx_http_header_out_t  ngx_http_headers_out[] = {
>      { ngx_string("Server"), offsetof(ngx_http_headers_out_t, server) },
>      { ngx_string("Date"), offsetof(ngx_http_headers_out_t, date) },
> @@ -151,6 +152,7 @@
> 
>      { ngx_null_string, 0 }
>  };
> ```
> 

It was used to support Apache LogFormat %{VARNAME}o syntax before 0.5.0.

-- 
Sergey Kandaurov

From mdounin at mdounin.ru  Thu Jul 20 16:32:51 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 20 Jul 2023 19:32:51 +0300
Subject: [PATCH] Contrib: vim syntax, update core and 3rd party module
 directives.
In-Reply-To: <5f76c221-445a-ff97-f9b2-04430b0902d1@csdoc.com>
References: <5f76c221-445a-ff97-f9b2-04430b0902d1@csdoc.com>
Message-ID: <ZLlhs6M2sHZWdp3k@mdounin.ru>

Hello!

On Thu, Jul 20, 2023 at 12:50:19AM +0300, Gena Makhomed wrote:

> # HG changeset patch
> # User Gena Makhomed <gmm at csdoc.com>
> # Date 1689803114 -10800
> #      Thu Jul 20 00:45:14 2023 +0300
> # Node ID dee0e911380d18773dad06d45b845a124474f12a
> # Parent  c209dc4eed1737261c97a044d23f5fb9afbd9d45
> Contrib: vim syntax, update core and 3rd party module directives.
> 
> List of 3rd party modules github repositories are obtained from
> https://github.com/freebsd/freebsd-ports/blob/main/www/nginx-devel/Makefile.extmod
> 
> diff -r c209dc4eed17 -r dee0e911380d contrib/vim/syntax/nginx.vim
> --- a/contrib/vim/syntax/nginx.vim	Wed Jul 19 05:09:23 2023 +0300
> +++ b/contrib/vim/syntax/nginx.vim	Thu Jul 20 00:45:14 2023 +0300
> @@ -65,43 +65,44 @@
>       \ contained
>       \ nextgroup=@ngxListenParams skipwhite skipempty
>   syn keyword ngxListenOptions contained
> -    \ default_server ssl http2 proxy_protocol
> +    \ default_server ssl quic proxy_protocol
>       \ setfib fastopen backlog rcvbuf sndbuf accept_filter deferred bind
>       \ ipv6only reuseport so_keepalive
>       \ nextgroup=@ngxListenParams skipwhite skipempty
>   syn keyword ngxListenOptionsDeprecated contained
> -    \ spdy
> +    \ http2
>       \ nextgroup=@ngxListenParams skipwhite skipempty
>   syn cluster ngxListenParams
>       \ contains=ngxListenParam,ngxListenString,ngxListenComment
>       \ add=ngxListenOptions,ngxListenOptionsDeprecated
> 
> -syn keyword ngxDirectiveBlock contained http
> -syn keyword ngxDirectiveBlock contained stream
> -syn keyword ngxDirectiveBlock contained mail
> +syn keyword ngxDirectiveBlock contained charset_map
>   syn keyword ngxDirectiveBlock contained events
> -syn keyword ngxDirectiveBlock contained server
> -syn keyword ngxDirectiveBlock contained types
> +syn keyword ngxDirectiveBlock contained geo
> +syn keyword ngxDirectiveBlock contained http
> +syn keyword ngxDirectiveBlock contained if
> +syn keyword ngxDirectiveBlock contained limit_except
>   syn keyword ngxDirectiveBlock contained location
> -syn keyword ngxDirectiveBlock contained upstream
> -syn keyword ngxDirectiveBlock contained charset_map
> -syn keyword ngxDirectiveBlock contained limit_except
> -syn keyword ngxDirectiveBlock contained if
> -syn keyword ngxDirectiveBlock contained geo
> +syn keyword ngxDirectiveBlock contained mail
>   syn keyword ngxDirectiveBlock contained map
> +syn keyword ngxDirectiveBlock contained match
> +syn keyword ngxDirectiveBlock contained otel_exporter
> +syn keyword ngxDirectiveBlock contained server
>   syn keyword ngxDirectiveBlock contained split_clients
> -syn keyword ngxDirectiveBlock contained match
> +syn keyword ngxDirectiveBlock contained stream
> +syn keyword ngxDirectiveBlock contained types
> +syn keyword ngxDirectiveBlock contained upstream

I don't think the order change here is a good idea.

(Also, it might be a good idea to keep directives introduced by 
commercial extensions separately.)

> 
> +syn keyword ngxDirectiveImportant contained fastcgi_pass
>   syn keyword ngxDirectiveImportant contained include
> -syn keyword ngxDirectiveImportant contained root
> -syn keyword ngxDirectiveImportant contained server_name
>   syn keyword ngxDirectiveImportant contained internal
> +syn keyword ngxDirectiveImportant contained memcached_pass
>   syn keyword ngxDirectiveImportant contained proxy_pass
> -syn keyword ngxDirectiveImportant contained memcached_pass
> -syn keyword ngxDirectiveImportant contained fastcgi_pass
> +syn keyword ngxDirectiveImportant contained root
>   syn keyword ngxDirectiveImportant contained scgi_pass
> +syn keyword ngxDirectiveImportant contained server_name
> +syn keyword ngxDirectiveImportant contained try_files
>   syn keyword ngxDirectiveImportant contained uwsgi_pass
> -syn keyword ngxDirectiveImportant contained try_files

Same here.

> 
>   syn keyword ngxDirectiveControl contained break
>   syn keyword ngxDirectiveControl contained return
> @@ -111,25 +112,24 @@
>   syn keyword ngxDirectiveError contained error_page
>   syn keyword ngxDirectiveError contained post_action
> 
> -syn keyword ngxDirectiveDeprecated contained proxy_downstream_buffer
> -syn keyword ngxDirectiveDeprecated contained proxy_upstream_buffer
> -syn keyword ngxDirectiveDeprecated contained ssl
>   syn keyword ngxDirectiveDeprecated contained http2_idle_timeout
>   syn keyword ngxDirectiveDeprecated contained http2_max_field_size
>   syn keyword ngxDirectiveDeprecated contained http2_max_header_size
>   syn keyword ngxDirectiveDeprecated contained http2_max_requests
>   syn keyword ngxDirectiveDeprecated contained http2_recv_timeout
> +syn keyword ngxDirectiveDeprecated contained proxy_downstream_buffer
> +syn keyword ngxDirectiveDeprecated contained proxy_upstream_buffer

Same here.

> 
>   syn keyword ngxDirective contained absolute_redirect
> +syn keyword ngxDirective contained acceptex_read
>   syn keyword ngxDirective contained accept_mutex
>   syn keyword ngxDirective contained accept_mutex_delay
> -syn keyword ngxDirective contained acceptex_read
>   syn keyword ngxDirective contained access_log
>   syn keyword ngxDirective contained add_after_body
>   syn keyword ngxDirective contained add_before_body
>   syn keyword ngxDirective contained add_header
> +syn keyword ngxDirective contained addition_types
>   syn keyword ngxDirective contained add_trailer
> -syn keyword ngxDirective contained addition_types
>   syn keyword ngxDirective contained aio
>   syn keyword ngxDirective contained aio_write
>   syn keyword ngxDirective contained alias

It looks like order changes here (and in multiple other places) 
are caused by a changed alphabetical order in the tooling being 
used, which now places "_" after [a-z].

Apart from being an unneeded change, this does not look like a 
correct order to me: it generally contradicts order of characters 
in ASCII, and also doesn't play well with how nginx uses "_" (as 
"foo" and "foo_bar" are expected to be close to each other, and 
both before "foobazz").

[...]

-- 
Maxim Dounin
http://mdounin.ru/

From gmm at csdoc.com  Sat Jul 22 11:56:22 2023
From: gmm at csdoc.com (Gena Makhomed)
Date: Sat, 22 Jul 2023 14:56:22 +0300
Subject: [PATCH] Contrib: vim syntax, update core and 3rd party module
 directives.
In-Reply-To: <ZLlhs6M2sHZWdp3k@mdounin.ru>
References: <5f76c221-445a-ff97-f9b2-04430b0902d1@csdoc.com>
 <ZLlhs6M2sHZWdp3k@mdounin.ru>
Message-ID: <7cd58e2b-b247-6230-76a9-21bf1cdeca1d@csdoc.com>

On 20.07.2023 19:32, Maxim Dounin wrote:

 > (Also, it might be a good idea to keep directives introduced by
 > commercial extensions separately.)

In addition to ngxDirectiveBlock and ngxDirective I should
introduce ngxDirectiveBlockCommercial and ngxDirectiveCommercial
and [allow users to] use different color for commercial directives
of "nginx" and "nginx block" types ?

Or I should just make separation of commercial directives
in the contrib/vim/syntax/nginx.vim file by grouping,
spaces and comments, without any distinguish
between commercial and open-source directives
in the user-visible level? Something like this:

# nginx directives

syn keyword ngxDirectiveBlock contained http
syn keyword ngxDirectiveBlock contained stream
...
syn keyword ngxDirectiveBlock contained match

...

syn keyword ngxDirective contained absolute_redirect
syn keyword ngxDirective contained accept_mutex
...
syn keyword ngxDirective contained zone_sync_ssl_verify_depth
syn keyword ngxDirective contained zone_sync_timeout


# nginx-plus commercial extensions directives

syn keyword ngxDirectiveBlock contained otel_exporter

...
syn keyword ngxDirective contained internal_redirect
syn keyword ngxDirective contained mqtt
syn keyword ngxDirective contained mqtt_preread
syn keyword ngxDirective contained mqtt_rewrite_buffer_size
syn keyword ngxDirective contained mqtt_set_connect
syn keyword ngxDirective contained otel_service_name
syn keyword ngxDirective contained otel_span_attr
syn keyword ngxDirective contained otel_span_name
syn keyword ngxDirective contained otel_trace
syn keyword ngxDirective contained otel_trace_context
...


>>    syn keyword ngxDirective contained add_header
>> +syn keyword ngxDirective contained addition_types
>>    syn keyword ngxDirective contained add_trailer
>> -syn keyword ngxDirective contained addition_types

> It looks like order changes here (and in multiple other places)
> are caused by a changed alphabetical order in the tooling being
> used, which now places "_" after [a-z].

As I understand - it just ignore "_", placing "addition_types"
after "add_header" but before "add_trailer",
because of order "h" < "i" < "t".

> Apart from being an unneeded change, this does not look like a
> correct order to me: it generally contradicts order of characters
> in ASCII, and also doesn't play well with how nginx uses "_" (as
> "foo" and "foo_bar" are expected to be close to each other, and
> both before "foobazz").

tooling used - is unix command line utility sort, via vim syntax:

:'<,'>!sort

for sorting selected block of lines.

in the system, Rocky Linux release 8.x, such settings:

LANG=en_US.UTF-8

so, in this case:

LC_COLLATE=en_US.UTF-8

and as I understand, this is not vim bug or sort utility bug,
this is en_US.UTF-8 collation bug - I don't know hot to fix this bug.

In future I will use

LC_ALL=C
LC_COLLATE=C

to sort lines in the ngxDirective`s block
via external unix command line sort tool,
or will use internal vim sort command via

:'<,'>sort

As I understand, - I should sort directives only
in ngxDirective and ngxDirectiveThirdParty blocks
and leave all other blocks in the existing "random" order?

Or I should not sort any blocks of lines
and I should just append new lines to end
of the each block, to minimize the patch size?

Something like this:

...
syn keyword ngxDirective contained zone_sync_ssl_verify
syn keyword ngxDirective contained zone_sync_ssl_verify_depth
syn keyword ngxDirective contained zone_sync_timeout
syn keyword ngxDirective contained http2
syn keyword ngxDirective contained http3
syn keyword ngxDirective contained http3_hq
syn keyword ngxDirective contained http3_max_concurrent_streams
syn keyword ngxDirective contained http3_stream_buffer_size
syn keyword ngxDirective contained js_preload_object
syn keyword ngxDirective contained js_shared_dict_zone
syn keyword ngxDirective contained quic_active_connection_id_limit
syn keyword ngxDirective contained quic_bpf
syn keyword ngxDirective contained quic_gso
syn keyword ngxDirective contained quic_host_key
syn keyword ngxDirective contained quic_retry

?

-- 
Best regards,
  Gena

From mdounin at mdounin.ru  Mon Jul 24 01:01:27 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 24 Jul 2023 04:01:27 +0300
Subject: [PATCH] Contrib: vim syntax, update core and 3rd party module
 directives.
In-Reply-To: <7cd58e2b-b247-6230-76a9-21bf1cdeca1d@csdoc.com>
References: <5f76c221-445a-ff97-f9b2-04430b0902d1@csdoc.com>
 <ZLlhs6M2sHZWdp3k@mdounin.ru>
 <7cd58e2b-b247-6230-76a9-21bf1cdeca1d@csdoc.com>
Message-ID: <ZL3NZ3xrWy4B/NTn@mdounin.ru>

Hello!

On Sat, Jul 22, 2023 at 02:56:22PM +0300, Gena Makhomed wrote:

> On 20.07.2023 19:32, Maxim Dounin wrote:
> 
>  > (Also, it might be a good idea to keep directives introduced by
>  > commercial extensions separately.)
> 
> In addition to ngxDirectiveBlock and ngxDirective I should
> introduce ngxDirectiveBlockCommercial and ngxDirectiveCommercial
> and [allow users to] use different color for commercial directives
> of "nginx" and "nginx block" types ?
> 
> Or I should just make separation of commercial directives
> in the contrib/vim/syntax/nginx.vim file by grouping,
> spaces and comments, without any distinguish
> between commercial and open-source directives
> in the user-visible level? Something like this:
> 
> # nginx directives
> 
> syn keyword ngxDirectiveBlock contained http
> syn keyword ngxDirectiveBlock contained stream
> ...
> syn keyword ngxDirectiveBlock contained match
> 
> ...
> 
> syn keyword ngxDirective contained absolute_redirect
> syn keyword ngxDirective contained accept_mutex
> ...
> syn keyword ngxDirective contained zone_sync_ssl_verify_depth
> syn keyword ngxDirective contained zone_sync_timeout
> 
> 
> # nginx-plus commercial extensions directives
> 
> syn keyword ngxDirectiveBlock contained otel_exporter
> 
> ...
> syn keyword ngxDirective contained internal_redirect
> syn keyword ngxDirective contained mqtt
> syn keyword ngxDirective contained mqtt_preread
> syn keyword ngxDirective contained mqtt_rewrite_buffer_size
> syn keyword ngxDirective contained mqtt_set_connect
> syn keyword ngxDirective contained otel_service_name
> syn keyword ngxDirective contained otel_span_attr
> syn keyword ngxDirective contained otel_span_name
> syn keyword ngxDirective contained otel_trace
> syn keyword ngxDirective contained otel_trace_context
> ...

I think just grouping as suggested would be enough.

> >>    syn keyword ngxDirective contained add_header
> >> +syn keyword ngxDirective contained addition_types
> >>    syn keyword ngxDirective contained add_trailer
> >> -syn keyword ngxDirective contained addition_types
> 
> > It looks like order changes here (and in multiple other places)
> > are caused by a changed alphabetical order in the tooling being
> > used, which now places "_" after [a-z].
> 
> As I understand - it just ignore "_", placing "addition_types"
> after "add_header" but before "add_trailer",
> because of order "h" < "i" < "t".
> 
> > Apart from being an unneeded change, this does not look like a
> > correct order to me: it generally contradicts order of characters
> > in ASCII, and also doesn't play well with how nginx uses "_" (as
> > "foo" and "foo_bar" are expected to be close to each other, and
> > both before "foobazz").
> 
> tooling used - is unix command line utility sort, via vim syntax:
> 
> :'<,'>!sort
> 
> for sorting selected block of lines.
> 
> in the system, Rocky Linux release 8.x, such settings:
> 
> LANG=en_US.UTF-8
> 
> so, in this case:
> 
> LC_COLLATE=en_US.UTF-8
> 
> and as I understand, this is not vim bug or sort utility bug,
> this is en_US.UTF-8 collation bug - I don't know hot to fix this bug.

Indeed, it seems to ignore "_" with LC_COLLATE=en_US.UTF-8 on 
Linux (checked on Ubuntu 22.04).

> In future I will use
> 
> LC_ALL=C
> LC_COLLATE=C
> 
> to sort lines in the ngxDirective`s block
> via external unix command line sort tool,
> or will use internal vim sort command via
> 
> :'<,'>sort

Yes, that would be awesome.

> As I understand, - I should sort directives only
> in ngxDirective and ngxDirectiveThirdParty blocks
> and leave all other blocks in the existing "random" order?
> 
> Or I should not sort any blocks of lines
> and I should just append new lines to end
> of the each block, to minimize the patch size?

I think it make sense to keep sorting where it is already in 
place, but preserve manual ordering in groups where manual 
ordering is already used (such as importance-based ordering in 
ngxDirectiveBlock and ngxDirectiveImportant blocks, and date-based 
in the ngxDirectiveDeprecated block).

[...]

-- 
Maxim Dounin
http://mdounin.ru/

From jordanc.carter at outlook.com  Mon Jul 24 04:24:04 2023
From: jordanc.carter at outlook.com (J Carter)
Date: Mon, 24 Jul 2023 05:24:04 +0100
Subject: [PATCH] Added debug_random directive
In-Reply-To: <LO0P123MB63193421DA656E482B9357FF9E38A@LO0P123MB6319.GBRP123.PROD.OUTLOOK.COM>
References: <LO0P123MB63194D794B1808A023A7FC9C9E35A@LO0P123MB6319.GBRP123.PROD.OUTLOOK.COM>
 <LO0P123MB63193421DA656E482B9357FF9E38A@LO0P123MB6319.GBRP123.PROD.OUTLOOK.COM>
Message-ID: <LO0P123MB63190031849B0A8B468A1B219E02A@LO0P123MB6319.GBRP123.PROD.OUTLOOK.COM>

On Tue, 18 Jul 2023 03:36:02 +0000
J Carter <jordanc.carter at outlook.com> wrote:

> On Sat, 15 Jul 2023 03:48:25 +0000
> J Carter <jordanc.carter at outlook.com> wrote:
> 
> > # HG changeset patch
> > # User J Carter <jordanc.carter at outlook.com>
> > # Date 1689391559 -3600
> > #      Sat Jul 15 04:25:59 2023 +0100
> > # Node ID b1ea0a60417e547513654bf9d6bb79714865c780
> > # Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
> > Added debug_random directive
> > 
> > This directive enforces for EITHER a percentage of total connections
> > OR a percentage of connections matched by debug_connection CIDRs
> > to have debug logging enabled.
> > 
> > This is useful for debugging when nginx is under high load
> > (production) - where debugging all connections is not possible without
> > disrupting traffic.
> > 
> > This directive takes a value between 0.00%-100.00% exclusive.
> > 
> 
> # HG changeset patch
> # User J Carter <jordanc.carter at outlook.com>
> # Date 1689649226 -3600
> #      Tue Jul 18 04:00:26 2023 +0100
> # Node ID 87f6f95e0385e6cd37354979ea61cc2435deb430
> # Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
> Added debug_random directive
> 
> Rework of previous patch. Fixed several bugs.
> 
> Example usage:
> 
> events {
>      worker_connections  1024;
>      #if uncommented, the percentage applies to connection from lo.
>      #debug_connection    127.0.0.0/8;
>      debug_random        1%;
> }
> 
# HG changeset patch
# User J Carter <jordanc.carter at outlook.com>
# Date 1690171706 -3600
#      Mon Jul 24 05:08:26 2023 +0100
# Node ID ea91b9aa69d8ce9dc9878209a83b7d538e6bc7e1
# Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
Added debug_random directive

More bug fixes and style changes.

diff -r 77c1418916f7 -r ea91b9aa69d8 src/event/ngx_event.c
--- a/src/event/ngx_event.c	Thu Jun 08 14:58:01 2023 +0400
+++ b/src/event/ngx_event.c	Mon Jul 24 05:08:26 2023 +0100
@@ -30,6 +30,8 @@
 static char *ngx_event_use(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
 static char *ngx_event_debug_connection(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+static char *ngx_event_debug_random(ngx_conf_t *cf, ngx_command_t *cmd,
+    void *conf);
 
 static void *ngx_event_core_create_conf(ngx_cycle_t *cycle);
 static char *ngx_event_core_init_conf(ngx_cycle_t *cycle, void *conf);
@@ -162,6 +164,13 @@
       0,
       NULL },
 
+      { ngx_string("debug_random"),
+      NGX_EVENT_CONF|NGX_CONF_TAKE1,
+      ngx_event_debug_random,
+      0,
+      0,
+      NULL },
+
       ngx_null_command
 };
 
@@ -496,6 +505,7 @@
     size_t               size, cl;
     ngx_shm_t            shm;
     ngx_time_t          *tp;
+    ngx_cidr_t          *cidr;
     ngx_core_conf_t     *ccf;
     ngx_event_conf_t    *ecf;
 
@@ -507,6 +517,33 @@
                       "using the \"%s\" event method", ecf->name);
     }
 
+    if (ecf->debug_connection.nelts == 0
+        && ecf->debug_scaled_pct > 0)
+    {
+        cidr = ngx_array_push(&ecf->debug_connection);
+        if (cidr == NULL) {
+            return NGX_ERROR;
+        }
+        /*0.0.0.0/0*/
+        ngx_memzero(cidr, sizeof(ngx_cidr_t));
+        cidr->family = AF_INET;
+
+#ifdef NGX_HAVE_INET6
+        cidr = ngx_array_push(&ecf->debug_connection);
+        if (cidr == NULL) {
+            return NGX_ERROR;
+        }
+        /*::/0*/
+        ngx_memzero(cidr, sizeof(ngx_cidr_t));
+        cidr->family = AF_INET6;
+#endif
+
+    } else if (ecf->debug_connection.nelts > 0
+               && ecf->debug_scaled_pct == 0)
+    {
+        ecf->debug_scaled_pct = NGX_MAX_INT32_VALUE;
+    }
+
     ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
 
     ngx_timer_resolution = ccf->timer_resolution;
@@ -1254,6 +1291,55 @@
 }
 
 
+static char *
+ngx_event_debug_random(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+#if (NGX_DEBUG)
+    ngx_event_conf_t  *ecf = conf;
+
+    u_char               *c;
+    ngx_int_t             pct;
+    ngx_uint_t            len;
+    ngx_str_t            *value;
+
+    if (ecf->debug_scaled_pct != NGX_CONF_UNSET_UINT) {
+        return "is duplicate";
+    }
+
+    value = cf->args->elts;
+    c = value[1].data;
+    len = value[1].len;
+
+    if (c[len-1] != '%') {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                               "%V missing '%%'",
+                               &value[1]);
+        return NGX_CONF_ERROR;
+    }
+
+    pct = ngx_atofp(c, len-1, 2);
+
+    if (pct == NGX_ERROR || pct == 0 || pct > 9999) {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                               "%V is an invalid value",
+                               &value[1]);
+        return NGX_CONF_ERROR;
+    }
+
+    ecf->debug_scaled_pct = NGX_MAX_INT32_VALUE / 10000 * pct;
+
+#else
+
+    ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
+                       "\"debug_random\" is ignored, you need to rebuild "
+                       "nginx using --with-debug option to enable it");
+
+#endif
+
+    return NGX_CONF_OK;
+}
+
+
 static void *
 ngx_event_core_create_conf(ngx_cycle_t *cycle)
 {
@@ -1279,6 +1365,8 @@
         return NULL;
     }
 
+    ecf->debug_scaled_pct = NGX_CONF_UNSET_UINT;
+
 #endif
 
     return ecf;
@@ -1369,5 +1457,7 @@
     ngx_conf_init_value(ecf->accept_mutex, 0);
     ngx_conf_init_msec_value(ecf->accept_mutex_delay, 500);
 
+    ngx_conf_init_uint_value(ecf->debug_scaled_pct, 0);
+
     return NGX_CONF_OK;
 }
diff -r 77c1418916f7 -r ea91b9aa69d8 src/event/ngx_event.h
--- a/src/event/ngx_event.h	Thu Jun 08 14:58:01 2023 +0400
+++ b/src/event/ngx_event.h	Mon Jul 24 05:08:26 2023 +0100
@@ -438,6 +438,7 @@
     u_char       *name;
 
 #if (NGX_DEBUG)
+    ngx_uint_t    debug_scaled_pct;
     ngx_array_t   debug_connection;
 #endif
 } ngx_event_conf_t;
diff -r 77c1418916f7 -r ea91b9aa69d8 src/event/ngx_event_accept.c
--- a/src/event/ngx_event_accept.c	Thu Jun 08 14:58:01 2023 +0400
+++ b/src/event/ngx_event_accept.c	Mon Jul 24 05:08:26 2023 +0100
@@ -523,6 +523,7 @@
     struct sockaddr_in6  *sin6;
     ngx_uint_t            n;
 #endif
+    ngx_uint_t r = ngx_random();
 
     cidr = ecf->debug_connection.elts;
     for (i = 0; i < ecf->debug_connection.nelts; i++) {
@@ -548,6 +549,7 @@
 
 #if (NGX_HAVE_UNIX_DOMAIN)
         case AF_UNIX:
+            r = 0;
             break;
 #endif
 
@@ -561,7 +563,10 @@
             break;
         }
 
-        c->log->log_level = NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL;
+        if (r <= ecf->debug_scaled_pct) {
+            c->log->log_level = NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL;
+        }
+
         break;
 
     next:

-------------- next part --------------
A non-text attachment was scrubbed...
Name: hgexport
Type: application/octet-stream
Size: 5190 bytes
Desc: not available
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20230724/359540fb/attachment.obj>

From l.crilly at f5.com  Mon Jul 24 14:21:16 2023
From: l.crilly at f5.com (Liam Crilly)
Date: Mon, 24 Jul 2023 14:21:16 +0000
Subject: [PATCH 0 of 3] nginx.org usability enhancements
Message-ID: <BY3PR01MB667424E626632E5343465173A502A@BY3PR01MB6674.prod.exchangelabs.com>

Hello!

This is the first of a multiple patch series for nginx.org.

This changeset brings numerous usability enhancements while retaining the
familiar look and feel of the site.

The primary visual changes are:

*) Readability on mobile devices
   - The right-hand menu is not shown for narrow viewports. In this case a
     conventional 'hamburger' icon on the banner is used to show the menu.
   - Font sizes are more consistent, moving primarily to the rem metric.

*) Increased contrast and emphasis for code and configuration styles in the
   'pre' and 'code' blocks.

*) Dark mode honors the client's system preferences and is automatically
   applied.

The embedded stylesheet has been extracted into separate files to aid
ongoing development and maintenance. The special character layouts for the
redundant Hebrew and Chinese translations are preserved but have not
received further enhancements. The Russian and English stylesheets are
currently identical and will be consolidated in a future changeset. For the
timebeing, any changes to one must be copied to the other.

This changeset should apply cleanly without requiring configuration changes
to the underlying web server.

Cheers,
Liam.

From l.crilly at f5.com  Mon Jul 24 14:23:12 2023
From: l.crilly at f5.com (Liam Crilly)
Date: Mon, 24 Jul 2023 14:23:12 +0000
Subject: [PATCH 1 of 3] CSS as file.
In-Reply-To: <BY3PR01MB667424E626632E5343465173A502A@BY3PR01MB6674.prod.exchangelabs.com>
References: <BY3PR01MB667424E626632E5343465173A502A@BY3PR01MB6674.prod.exchangelabs.com>
Message-ID: <BY3PR01MB6674D0B67BA5296084AC63A3A502A@BY3PR01MB6674.prod.exchangelabs.com>

# HG changeset patch
# User Liam Crilly <liam.crilly at nginx.com>
# Date 1690207197 -3600
#      Mon Jul 24 14:59:57 2023 +0100
# Node ID df1cf98cf8f50eb1770d966aed583d21e481558b
# Parent  1f672755959a64aec3f0aeceab1dbdc13cb36414
CSS as file.

Extracts the inline styles from style.xsls into separate files for
ease of style-development and to take advantage of browser caching.
Hebrew and Chinese variants are preserved.

Stylesheets are located in a new top-level directory (/css) and are
compressed during the gzip process.

diff -r 1f672755959a -r df1cf98cf8f5 GNUmakefile
--- a/GNUmakefile	Thu Jul 06 18:12:53 2023 +0100
+++ b/GNUmakefile	Mon Jul 24 14:59:57 2023 +0100
@@ -1,6 +1,7 @@
 
 OUT =		libxslt
 TEXT =		text
+CSS =		css
 BANNER =	banner
 ZIP =		gzip
 NGINX_ORG =	/data/www/nginx.org
@@ -73,11 +74,12 @@
 		2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019	\
 		2020 2021 2022
 
-all:		news arx 404 $(LANGS)
+all:		news arx 404 css $(LANGS)
 
 news:		$(OUT)/index.html $(OUT)/index.rss
 arx:		$(foreach year,$(YEARS),$(OUT)/$(year).html)
 404:		$(OUT)/404.html
+css:		$(foreach f,$(wildcard css/*.css),$(OUT)/$(f))
 
 
 DIRIND_DEPS =
@@ -171,6 +173,9 @@
 	mkdir -p $(dir $@)
 	$(call XSLScript, $<, $@)
 
+$(OUT)/css/%.css:      css/%.css
+		mkdir -p $(dir $@)
+		cp $< $@
 
 genapi:
 	$(MAKE) -C yaml
@@ -332,6 +337,7 @@
 		$(addsuffix .gz, $(wildcard $(ZIP)/ru/CHANGES.ru-?.?))	\
 		$(addsuffix .gz, $(wildcard $(ZIP)/ru/CHANGES.ru-?.??))	\
 		$(addsuffix .gz, $(wildcard $(ZIP)/keys/*.key))		\
+		$(addsuffix .gz, $(wildcard $(ZIP)/css/*.css))		\
 
 	find $(ZIP) -type f ! -name '*.gz' -exec test \! -e {}.gz \; -print
 
diff -r 1f672755959a -r df1cf98cf8f5 css/style_cn.css
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/css/style_cn.css	Mon Jul 24 14:59:57 2023 +0100
@@ -0,0 +1,207 @@
+body {
+    background: white;
+    color: black;
+    font-family: sans-serif;
+    line-height: 1.4em;
+    text-align: center;
+    margin: 0;
+    padding: 0;
+}
+
+#banner {
+    background: black;
+    color: #F2F2F2;
+    line-height: 1.2em;
+    padding: .3em 0;
+    box-shadow: 0 5px 10px black;
+}
+
+#banner a {
+    color: #00B140;
+}
+
+#main {
+    text-align: left;
+    margin: 0 auto;
+    min-width: 32em;
+    max-width: 64em;
+}
+
+#menu {
+    float: right;
+    width: 11em;
+    padding: 0 .5em 1em .5em;
+    border-left: 2px solid #DDD;
+}
+
+ #content {
+    margin-right: 13.5em;
+    padding: 0 .2em 0 1.5em;
+}
+
+h1 {
+    display: block;
+    font-size: 3em;
+    text-align: left;
+    height: .7em;
+    margin: 0;
+    margin-bottom: .5em;
+}
+
+h1 img {
+    width: 100%;
+}
+
+h2 {
+    text-align: center;
+}
+
+p {
+    text-align: left;
+}
+
+table.news p {
+    margin-top: 0;
+}
+
+table.news td {
+    vertical-align: baseline;
+}
+
+table.news .date {
+    text-align: right;
+    padding-right: 0.5em;
+    white-space: nowrap;
+}
+
+table.donors td {
+    vertical-align: baseline;
+}
+
+table.donors li {
+    text-align: left;
+}
+
+div.directive {
+    background: #F2F2F2;
+    line-height: 1em;
+    margin: 1em 0 1em -1em;
+    padding: .7em .7em .7em 1em;
+    border-top: 2px solid #DDD;
+}
+
+div.directive th {
+    padding-left: 0;
+    padding-right: .5em;
+    vertical-align: baseline;
+    text-align: left;
+    font-weight: normal;
+}
+
+div.directive td {
+    vertical-align: baseline;
+}
+
+div.directive pre {
+    padding: 0;
+    margin: 0;
+}
+
+div.directive p {
+    margin: .5em 0 0 .1em;
+    font-size: .8em;
+}
+
+a.notrans {
+    color: gray;
+    text-decoration:none;
+}
+
+span.initial {
+    font-size: 200%;
+    float: left;
+    padding-right: 10pt;
+}
+
+ul, ol {
+    margin: .5em 0 1em 1em;
+    padding: 0 .5em;
+}
+
+ol {
+    list-style-position: inside;
+}
+
+li {
+    text-align: left;
+    padding: .5em 0 0 1px;
+}
+
+.compact li {
+    padding-top: 0;
+}
+
+dl {
+    margin: .5em 0 1em 0;
+}
+
+dt {
+    margin: .5em 0;
+}
+
+.compact dt {
+    margin-bottom: .2em;
+}
+
+dd {
+    margin-left: 1.5em;
+    padding-left: 1px;
+    text-align: left;
+}
+
+td.list {
+    background: #F2F2F2;
+}
+
+blockquote {
+    margin: 1em 0 1em 1em;
+    padding: .5em;
+}
+
+li blockquote, dd blockquote {
+    margin: .7em 0;
+}
+
+blockquote.note {
+    border: 1px dotted #999;
+    line-height: 1.2em;
+    text-align: left;
+}
+
+blockquote.example {
+    line-height: 1em;
+    border-left: 1px solid #BBB;
+}
+
+blockquote.example pre {
+    padding: 0;
+    margin: 0;
+}
+
+sup {
+    font-size: 50%;
+}
+
+.video {
+    position: relative;
+    padding-bottom: 56.25%;
+    overflow: hidden;
+}
+
+.video iframe, .video object, .video embed {
+    position: absolute;
+    top:0;
+    left:0;
+    width:100%;
+    height:100%;
+}
diff -r 1f672755959a -r df1cf98cf8f5 css/style_en.css
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/css/style_en.css	Mon Jul 24 14:59:57 2023 +0100
@@ -0,0 +1,207 @@
+body {
+    background: white;
+    color: black;
+    font-family: sans-serif;
+    line-height: 1.4em;
+    text-align: center;
+    margin: 0;
+    padding: 0;
+}
+
+#banner {
+    background: black;
+    color: #F2F2F2;
+    line-height: 1.2em;
+    padding: .3em 0;
+    box-shadow: 0 5px 10px black;
+}
+
+#banner a {
+    color: #00B140;
+}
+
+#main {
+    text-align: left;
+    margin: 0 auto;
+    min-width: 32em;
+    max-width: 64em;
+}
+
+#menu {
+    float: right;
+    width: 11em;
+    padding: 0 .5em 1em .5em;
+    border-left: 2px solid #DDD;
+}
+
+ #content {
+    margin-right: 13.5em;
+    padding: 0 .2em 0 1.5em;
+}
+
+h1 {
+    display: block;
+    font-size: 3em;
+    text-align: left;
+    height: .7em;
+    margin: 0;
+    margin-bottom: .5em;
+}
+
+h1 img {
+    width: 100%;
+}
+
+h2 {
+    text-align: center;
+}
+
+p {
+    text-align: justify;
+}
+
+table.news p {
+    margin-top: 0;
+}
+
+table.news td {
+    vertical-align: baseline;
+}
+
+table.news .date {
+    text-align: right;
+    padding-right: 0.5em;
+    white-space: nowrap;
+}
+
+table.donors td {
+    vertical-align: baseline;
+}
+
+table.donors li {
+    text-align: left;
+}
+
+div.directive {
+    background: #F2F2F2;
+    line-height: 1em;
+    margin: 1em 0 1em -1em;
+    padding: .7em .7em .7em 1em;
+    border-top: 2px solid #DDD;
+}
+
+div.directive th {
+    padding-left: 0;
+    padding-right: .5em;
+    vertical-align: baseline;
+    text-align: left;
+    font-weight: normal;
+}
+
+div.directive td {
+    vertical-align: baseline;
+}
+
+div.directive pre {
+    padding: 0;
+    margin: 0;
+}
+
+div.directive p {
+    margin: .5em 0 0 .1em;
+    font-size: .8em;
+}
+
+a.notrans {
+    color: gray;
+    text-decoration:none;
+}
+
+span.initial {
+    font-size: 200%;
+    float: left;
+    padding-right: 10pt;
+}
+
+ul, ol {
+    margin: .5em 0 1em 1em;
+    padding: 0 .5em;
+}
+
+ol {
+    list-style-position: inside;
+}
+
+li {
+    text-align: justify;
+    padding: .5em 0 0 1px;
+}
+
+.compact li {
+    padding-top: 0;
+}
+
+dl {
+    margin: .5em 0 1em 0;
+}
+
+dt {
+    margin: .5em 0;
+}
+
+.compact dt {
+    margin-bottom: .2em;
+}
+
+dd {
+    margin-left: 1.5em;
+    padding-left: 1px;
+    text-align: justify;
+}
+
+td.list {
+    background: #F2F2F2;
+}
+
+blockquote {
+    margin: 1em 0 1em 1em;
+    padding: .5em;
+}
+
+li blockquote, dd blockquote {
+    margin: .7em 0;
+}
+
+blockquote.note {
+    border: 1px dotted #999;
+    line-height: 1.2em;
+    text-align: justify;
+}
+
+blockquote.example {
+    line-height: 1em;
+    border-left: 1px solid #BBB;
+}
+
+blockquote.example pre {
+    padding: 0;
+    margin: 0;
+}
+
+sup {
+    font-size: 50%;
+}
+
+.video {
+    position: relative;
+    padding-bottom: 56.25%;
+    overflow: hidden;
+}
+
+.video iframe, .video object, .video embed {
+    position: absolute;
+    top:0;
+    left:0;
+    width:100%;
+    height:100%;
+}
diff -r 1f672755959a -r df1cf98cf8f5 css/style_he.css
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/css/style_he.css	Mon Jul 24 14:59:57 2023 +0100
@@ -0,0 +1,222 @@
+body {
+    background: white;
+    color: black;
+    font-family: sans-serif;
+    line-height: 1.4em;
+    text-align: center;
+    margin: 0;
+    padding: 0;
+}
+
+#banner {
+    background: black;
+    color: #F2F2F2;
+    line-height: 1.2em;
+    padding: .3em 0;
+    box-shadow: 0 5px 10px black;
+}
+
+#banner a {
+    color: #00B140;
+}
+
+#main {
+    text-align: left;
+    margin: 0 auto;
+    min-width: 32em;
+    max-width: 64em;
+}
+
+#menu {
+    float: right;
+    width: 11em;
+    padding: 0 .5em 1em .5em;
+    border-left: 2px solid #DDD;
+}
+
+ #content {
+    margin-right: 13.5em;
+    padding: 0 .2em 0 1.5em;
+}
+
+h1 {
+    display: block;
+    font-size: 3em;
+    text-align: left;
+    height: .7em;
+    margin: 0;
+    margin-bottom: .5em;
+}
+
+h1 img {
+    width: 100%;
+}
+
+h2 {
+    text-align: center;
+}
+
+p {
+    text-align: justify;
+}
+
+pre {
+    text-align: left;
+    direction: ltr;
+}
+
+code {
+    direction: ltr;
+    unicode-bidi: embed;
+}
+
+.ltr {
+    direction: ltr;
+    unicode-bidi: embed;
+}
+
+table.news p {
+    margin-top: 0;
+}
+
+table.news td {
+    vertical-align: baseline;
+}
+
+table.news .date {
+    text-align: right;
+    padding-right: 0.5em;
+    white-space: nowrap;
+}
+
+table.donors td {
+    vertical-align: baseline;
+}
+
+table.donors li {
+    text-align: left;
+}
+
+div.directive {
+    background: #F2F2F2;
+    line-height: 1em;
+    margin: 1em 0 1em -1em;
+    padding: .7em .7em .7em 1em;
+    border-top: 2px solid #DDD;
+}
+
+div.directive th {
+    padding-left: 0;
+    padding-right: .5em;
+    vertical-align: baseline;
+    text-align: left;
+    font-weight: normal;
+}
+
+div.directive td {
+    vertical-align: baseline;
+}
+
+div.directive pre {
+    padding: 0;
+    margin: 0;
+}
+
+div.directive p {
+    margin: .5em 0 0 .1em;
+    font-size: .8em;
+}
+
+a.notrans {
+    color: gray;
+    text-decoration:none;
+}
+
+span.initial {
+    font-size: 200%;
+    float: left;
+    padding-right: 10pt;
+}
+
+ul, ol {
+    margin: .5em 0 1em 1em;
+    padding: 0 .5em;
+}
+
+ol {
+    list-style-position: inside;
+}
+
+li {
+    text-align: justify;
+    padding: .5em 0 0 1px;
+}
+
+.compact li {
+    padding-top: 0;
+}
+
+dl {
+    margin: .5em 0 1em 0;
+}
+
+dt {
+    margin: .5em 0;
+}
+
+.compact dt {
+    margin-bottom: .2em;
+}
+
+dd {
+    margin-left: 1.5em;
+    padding-left: 1px;
+    text-align: justify;
+}
+
+td.list {
+    background: #F2F2F2;
+}
+
+blockquote {
+    margin: 1em 0 1em 1em;
+    padding: .5em;
+}
+
+li blockquote, dd blockquote {
+    margin: .7em 0;
+}
+
+blockquote.note {
+    border: 1px dotted #999;
+    line-height: 1.2em;
+    text-align: justify;
+}
+
+blockquote.example {
+    line-height: 1em;
+    border-left: 1px solid #BBB;
+}
+
+blockquote.example pre {
+    padding: 0;
+    margin: 0;
+}
+
+sup {
+    font-size: 50%;
+}
+
+.video {
+    position: relative;
+    padding-bottom: 56.25%;
+    overflow: hidden;
+}
+
+.video iframe, .video object, .video embed {
+    position: absolute;
+    top:0;
+    left:0;
+    width:100%;
+    height:100%;
+}
diff -r 1f672755959a -r df1cf98cf8f5 css/style_ru.css
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/css/style_ru.css	Mon Jul 24 14:59:57 2023 +0100
@@ -0,0 +1,207 @@
+body {
+    background: white;
+    color: black;
+    font-family: sans-serif;
+    line-height: 1.4em;
+    text-align: center;
+    margin: 0;
+    padding: 0;
+}
+
+#banner {
+    background: black;
+    color: #F2F2F2;
+    line-height: 1.2em;
+    padding: .3em 0;
+    box-shadow: 0 5px 10px black;
+}
+
+#banner a {
+    color: #00B140;
+}
+
+#main {
+    text-align: left;
+    margin: 0 auto;
+    min-width: 32em;
+    max-width: 64em;
+}
+
+#menu {
+    float: right;
+    width: 11em;
+    padding: 0 .5em 1em .5em;
+    border-left: 2px solid #DDD;
+}
+
+ #content {
+    margin-right: 13.5em;
+    padding: 0 .2em 0 1.5em;
+}
+
+h1 {
+    display: block;
+    font-size: 3em;
+    text-align: left;
+    height: .7em;
+    margin: 0;
+    margin-bottom: .5em;
+}
+
+h1 img {
+    width: 100%;
+}
+
+h2 {
+    text-align: center;
+}
+
+p {
+    text-align: justify;
+}
+
+table.news p {
+    margin-top: 0;
+}
+
+table.news td {
+    vertical-align: baseline;
+}
+
+table.news .date {
+    text-align: right;
+    padding-right: 0.5em;
+    white-space: nowrap;
+}
+
+table.donors td {
+    vertical-align: baseline;
+}
+
+table.donors li {
+    text-align: left;
+}
+
+div.directive {
+    background: #F2F2F2;
+    line-height: 1em;
+    margin: 1em 0 1em -1em;
+    padding: .7em .7em .7em 1em;
+    border-top: 2px solid #DDD;
+}
+
+div.directive th {
+    padding-left: 0;
+    padding-right: .5em;
+    vertical-align: baseline;
+    text-align: left;
+    font-weight: normal;
+}
+
+div.directive td {
+    vertical-align: baseline;
+}
+
+div.directive pre {
+    padding: 0;
+    margin: 0;
+}
+
+div.directive p {
+    margin: .5em 0 0 .1em;
+    font-size: .8em;
+}
+
+a.notrans {
+    color: gray;
+    text-decoration:none;
+}
+
+span.initial {
+    font-size: 200%;
+    float: left;
+    padding-right: 10pt;
+}
+
+ul, ol {
+    margin: .5em 0 1em 1em;
+    padding: 0 .5em;
+}
+
+ol {
+    list-style-position: inside;
+}
+
+li {
+    text-align: justify;
+    padding: .5em 0 0 1px;
+}
+
+.compact li {
+    padding-top: 0;
+}
+
+dl {
+    margin: .5em 0 1em 0;
+}
+
+dt {
+    margin: .5em 0;
+}
+
+.compact dt {
+    margin-bottom: .2em;
+}
+
+dd {
+    margin-left: 1.5em;
+    padding-left: 1px;
+    text-align: justify;
+}
+
+td.list {
+    background: #F2F2F2;
+}
+
+blockquote {
+    margin: 1em 0 1em 1em;
+    padding: .5em;
+}
+
+li blockquote, dd blockquote {
+    margin: .7em 0;
+}
+
+blockquote.note {
+    border: 1px dotted #999;
+    line-height: 1.2em;
+    text-align: justify;
+}
+
+blockquote.example {
+    line-height: 1em;
+    border-left: 1px solid #BBB;
+}
+
+blockquote.example pre {
+    padding: 0;
+    margin: 0;
+}
+
+sup {
+    font-size: 50%;
+}
+
+.video {
+    position: relative;
+    padding-bottom: 56.25%;
+    overflow: hidden;
+}
+
+.video iframe, .video object, .video embed {
+    position: absolute;
+    top:0;
+    left:0;
+    width:100%;
+    height:100%;
+}
diff -r 1f672755959a -r df1cf98cf8f5 xsls/style.xsls
--- a/xsls/style.xsls	Thu Jul 06 18:12:53 2023 +0100
+++ b/xsls/style.xsls	Mon Jul 24 14:59:57 2023 +0100
@@ -7,126 +7,8 @@
 
 X:template style (lang) {
 
-    <style type="text/css">!{ normalize-space('
-        body              { background:     white;
-                            color:          black;
-                            font-family:    sans-serif;
-                            line-height:    1.4em;
-                            text-align:     center;
-                            margin:         0;
-                            padding:        0; }
-        #banner           { background:     black;
-                            color:          #F2F2F2;
-                            line-height:    1.2em;
-                            padding:        .3em 0;
-                            box-shadow:     0 5px 10px black; }
-        #banner a         { color:          #00B140; }
-        #main             { text-align:     left;
-                            margin:         0 auto;
-                            min-width:      32em;
-                            max-width:      64em; }
-        #menu             { float:          right;
-                            width:          11em;
-                            padding:        0 .5em 1em .5em;
-                            border-left:    2px solid #DDD; }
-        #content          { margin-right:   13.5em;
-                            padding:        0 .2em 0 1.5em; }
-        h1                { display:        block;
-                            font-size:      3em;
-                            text-align:     left;
-                            height:         .7em;
-                            margin:         0;
-                            margin-bottom:  .5em; }
-        h1 img            { width:          100%; }
-        h2                { text-align:     center; }
-        p                 { text-align:     justify; }
-        table.news p      { margin-top:     0; }
-        table.news td     { vertical-align: baseline; }
-        table.news .date  { text-align:     right;
-                            padding-right:  0.5em;
-                            white-space:    nowrap; }
-        table.donors td   { vertical-align: baseline; }
-        table.donors li   { text-align:     left; }
-        div.directive     { background:     #F2F2F2;
-                            line-height:    1em;
-                            margin:         1em 0 1em -1em;
-                            padding:        .7em .7em .7em 1em;
-                            border-top:     2px solid #DDD; }
-        div.directive th  { padding-left:   0;
-                            padding-right:  .5em;
-                            vertical-align: baseline;
-                            text-align:     left;
-                            font-weight:    normal; }
-        div.directive td  { vertical-align: baseline; }
-        div.directive pre { padding:        0;
-                            margin:         0; }
-        div.directive p   { margin:         .5em 0 0 .1em;
-                            font-size:      .8em; }
-        a.notrans         { color:          gray;
-                            text-decoration:none; }
-        span.initial      { font-size:      200%;
-                            float:          left;
-                            padding-right:  10pt;}
-        ul, ol            { margin:         .5em 0 1em 1em;
-                            padding:        0 .5em; }
-        ol                { list-style-position: inside; }
-        li                { text-align:     justify;
-                            padding:        .5em 0 0 1px; }
-        .compact li       { padding-top:    0; }
-        dl                { margin:         .5em 0 1em 0; }
-        dt                { margin:         .5em 0; }
-        .compact dt       { margin-bottom:  .2em; }
-        dd                { margin-left:    1.5em;
-                            padding-left:   1px;
-                            text-align:     justify; }
-        td.list           { background:     #F2F2F2; }
-        blockquote        { margin:         1em 0 1em 1em;
-                            padding:        .5em; }
-        li blockquote, dd blockquote
-                          { margin:         .7em 0; }
-        blockquote.note   { border:         1px dotted #999;
-                            line-height:    1.2em;
-                            text-align:     justify; }
-        blockquote.example {
-                            line-height:    1em;
-                            border-left:    1px solid #BBB; }
-        blockquote.example pre {
-                            padding:        0;
-                            margin:         0; }
-        sup               { font-size:      50%; }
-        .video            { position: relative;
-                            padding-bottom: 56.25%;
-                            overflow: hidden; }
-        .video iframe,
-        .video object,
-        .video embed      { position: absolute;
-                            top:0;
-                            left:0;
-                            width:100%;
-                            height:100%; }
-        ') }
+    <link rel="stylesheet" href="/css/style_{$lang}.css"></link>
 
-    X:if "$lang = 'he'" {
-        !{ normalize-space('
-        pre               { text-align:     left;
-                            direction:      ltr; }
-        code              { direction:      ltr;
-                            unicode-bidi:   embed; }
-        .ltr              { direction:      ltr;
-                            unicode-bidi:   embed; }
-        ') }
-    }
-
-    X:if "$lang = 'cn'" {
-        !{ normalize-space('
-        p                 { text-align:     left; }
-        li                { text-align:     left; }
-        dd                { text-align:     left; }
-        blockquote.note   { text-align:     left; }
-        ') }
-    }
-
-    </style>
 }
 
 }

From l.crilly at f5.com  Mon Jul 24 14:25:16 2023
From: l.crilly at f5.com (Liam Crilly)
Date: Mon, 24 Jul 2023 14:25:16 +0000
Subject: [PATCH 2 of 3] Responsive menu.
In-Reply-To: <BY3PR01MB6674D0B67BA5296084AC63A3A502A@BY3PR01MB6674.prod.exchangelabs.com>
References: <BY3PR01MB667424E626632E5343465173A502A@BY3PR01MB6674.prod.exchangelabs.com>
 <BY3PR01MB6674D0B67BA5296084AC63A3A502A@BY3PR01MB6674.prod.exchangelabs.com>
Message-ID: <BY3PR01MB6674B4CA03F27E73F9D3A0F7A502A@BY3PR01MB6674.prod.exchangelabs.com>

# HG changeset patch
# User Liam Crilly <liam.crilly at nginx.com>
# Date 1690207285 -3600
#      Mon Jul 24 15:01:25 2023 +0100
# Node ID c4018ab31dc52632f470298fcc7cece1fd8f57b9
# Parent  df1cf98cf8f50eb1770d966aed583d21e481558b
Responsive menu.

Nav menu is now hidden on narrow displays, collapsing to a "hamburger"
menu icon in the banner. Menu items are now an unordered list instead of
simple text markup.

diff -r df1cf98cf8f5 -r c4018ab31dc5 css/style_en.css
--- a/css/style_en.css	Mon Jul 24 14:59:57 2023 +0100
+++ b/css/style_en.css	Mon Jul 24 15:01:25 2023 +0100
@@ -29,10 +29,22 @@
 
 #menu {
     float: right;
-    width: 11em;
-    padding: 0 .5em 1em .5em;
+    width: 13em;
+    padding: 0;
     border-left: 2px solid #DDD;
 }
+.nav ul{
+    margin:0;
+    padding:20px;
+}
+.nav h1{
+    padding:0 20px;
+}
+.nav ul li{
+    list-style-type:none;
+    padding:0;
+    margin:0;
+}
 
  #content {
     margin-right: 13.5em;
@@ -43,9 +55,7 @@
     display: block;
     font-size: 3em;
     text-align: left;
-    height: .7em;
-    margin: 0;
-    margin-bottom: .5em;
+    margin: 0.5rem 0 0 0;
 }
 
 h1 img {
@@ -205,3 +215,125 @@
     width:100%;
     height:100%;
 }
+
+ at media screen and (max-width:768px) {
+    #main {
+        padding: 20px;
+        min-width: inherit;
+    }
+    #main #content {
+        width: 100%!important;
+        padding: 0;
+        border: none;
+    }
+
+    #banner-content {
+        max-width: 70vw;
+    }
+
+    #menu {
+        text-align: left;
+    }
+
+    /* Menu Mobile */
+    :root {
+        --white: #f9f9f9;
+        --black: #000;
+        --gray: #85888C;
+        --green: #b6d7a8;
+        color-scheme: light dark;
+    } /* variables*/
+
+    /* Nav menu */
+    .nav {
+        width: 15rem;
+        height: 100%;
+        max-height: 0;
+        position: fixed;
+        top: 50px;
+        right: 0;
+        border-left: 1px solid #909090;
+        background-color: var(--white);
+        overflow: hidden;
+        transition: .5s ease-in-out;
+    }
+    .nav ul {
+        margin: 0;
+        padding: 20px;
+    }
+    .nav h1 {
+        padding: 0 20px;
+    }
+    .nav ul li {
+        list-style-type: none;
+        padding:0;
+        margin:0;
+    }
+
+    .hamb {
+        cursor: pointer;
+        float: right;
+        position: absolute;
+        top: 0;
+        right: 20px;
+        height: 30px;
+        width: 70px;
+        padding-left: 20px;
+        padding-top: 20px;
+        z-index: 1100;
+    }
+    .hamb-line {
+        background: var(--green);
+        display: block;
+        height: 5px;
+        position: relative;
+        width: 44px;
+        border-radius:3px;
+
+    }
+    .hamb-line::before,.hamb-line::after {
+        background: var(--green);
+        content: '';
+        display: block;
+        height: 100%;
+        position: absolute;
+        transition: all .2s ease-in-out;
+        width: 100%;
+        border-radius:3px;
+    }
+    .hamb-line::before{
+        top: 10px;
+    }
+    .hamb-line::after{
+        top: -10px;
+    }
+    .side-menu {
+        display: none;
+    } /* Hide checkbox */
+    .side-menu:checked ~ .nav{
+        max-height: 100%;
+        top: 50px;
+
+    }
+    .side-menu:checked ~ .hamb .hamb-line {
+        background: transparent;
+    }
+    .side-menu:checked ~ .hamb .hamb-line::before {
+        transform: rotate(-45deg);
+        top: 0;
+    }
+    .side-menu:checked ~ .hamb .hamb-line::after {
+        transform: rotate(45deg);
+        top: 0;
+    }
+
+    code {
+        white-space: pre-line;
+    }
+}
+
+ at media screen and (min-width:768px){
+    .side-menu,.hamb-line {
+        display: none;
+    }
+}
diff -r df1cf98cf8f5 -r c4018ab31dc5 css/style_ru.css
--- a/css/style_ru.css	Mon Jul 24 14:59:57 2023 +0100
+++ b/css/style_ru.css	Mon Jul 24 15:01:25 2023 +0100
@@ -29,10 +29,22 @@
 
 #menu {
     float: right;
-    width: 11em;
-    padding: 0 .5em 1em .5em;
+    width: 13em;
+    padding: 0;
     border-left: 2px solid #DDD;
 }
+.nav ul{
+    margin:0;
+    padding:20px;
+}
+.nav h1{
+    padding:0 20px;
+}
+.nav ul li{
+    list-style-type:none;
+    padding:0;
+    margin:0;
+}
 
  #content {
     margin-right: 13.5em;
@@ -43,9 +55,7 @@
     display: block;
     font-size: 3em;
     text-align: left;
-    height: .7em;
-    margin: 0;
-    margin-bottom: .5em;
+    margin: 0.5rem 0 0 0;
 }
 
 h1 img {
@@ -205,3 +215,125 @@
     width:100%;
     height:100%;
 }
+
+ at media screen and (max-width:768px) {
+    #main {
+        padding: 20px;
+        min-width: inherit;
+    }
+    #main #content {
+        width: 100%!important;
+        padding: 0;
+        border: none;
+    }
+
+    #banner-content {
+        max-width: 70vw;
+    }
+
+    #menu {
+        text-align: left;
+    }
+
+    /* Menu Mobile */
+    :root {
+        --white: #f9f9f9;
+        --black: #000;
+        --gray: #85888C;
+        --green: #b6d7a8;
+        color-scheme: light dark;
+    } /* variables*/
+
+    /* Nav menu */
+    .nav {
+        width: 15rem;
+        height: 100%;
+        max-height: 0;
+        position: fixed;
+        top: 50px;
+        right: 0;
+        border-left: 1px solid #909090;
+        background-color: var(--white);
+        overflow: hidden;
+        transition: .5s ease-in-out;
+    }
+    .nav ul {
+        margin: 0;
+        padding: 20px;
+    }
+    .nav h1 {
+        padding: 0 20px;
+    }
+    .nav ul li {
+        list-style-type: none;
+        padding:0;
+        margin:0;
+    }
+
+    .hamb {
+        cursor: pointer;
+        float: right;
+        position: absolute;
+        top: 0;
+        right: 20px;
+        height: 30px;
+        width: 70px;
+        padding-left: 20px;
+        padding-top: 20px;
+        z-index: 1100;
+    }
+    .hamb-line {
+        background: var(--green);
+        display: block;
+        height: 5px;
+        position: relative;
+        width: 44px;
+        border-radius:3px;
+
+    }
+    .hamb-line::before,.hamb-line::after {
+        background: var(--green);
+        content: '';
+        display: block;
+        height: 100%;
+        position: absolute;
+        transition: all .2s ease-in-out;
+        width: 100%;
+        border-radius:3px;
+    }
+    .hamb-line::before{
+        top: 10px;
+    }
+    .hamb-line::after{
+        top: -10px;
+    }
+    .side-menu {
+        display: none;
+    } /* Hide checkbox */
+    .side-menu:checked ~ .nav{
+        max-height: 100%;
+        top: 50px;
+
+    }
+    .side-menu:checked ~ .hamb .hamb-line {
+        background: transparent;
+    }
+    .side-menu:checked ~ .hamb .hamb-line::before {
+        transform: rotate(-45deg);
+        top: 0;
+    }
+    .side-menu:checked ~ .hamb .hamb-line::after {
+        transform: rotate(45deg);
+        top: 0;
+    }
+
+    code {
+        white-space: pre-line;
+    }
+}
+
+ at media screen and (min-width:768px){
+    .side-menu,.hamb-line {
+        display: none;
+    }
+}
diff -r df1cf98cf8f5 -r c4018ab31dc5 xsls/banner.xsls
--- a/xsls/banner.xsls	Mon Jul 24 14:59:57 2023 +0100
+++ b/xsls/banner.xsls	Mon Jul 24 15:01:25 2023 +0100
@@ -21,7 +21,7 @@
             fetch("!banner_link ()")
                 .then((response) => response.text())
                 .then((resp) => \{
-                    document.getElementById("banner").innerHTML = resp;
+                    document.getElementById("banner-content").innerHTML = resp;
                 \})
                 .catch((error) => \{
                     console.warn(error);
diff -r df1cf98cf8f5 -r c4018ab31dc5 xsls/body.xsls
--- a/xsls/body.xsls	Mon Jul 24 14:59:57 2023 +0100
+++ b/xsls/body.xsls	Mon Jul 24 15:01:25 2023 +0100
@@ -18,20 +18,27 @@
     <body>
 
     <div id="banner">
+        <div id="banner-content"></div>
     </div>
 
     <div id="main">
-    <div id="menu">
+    <div id="menu" >
+        <input class="side-menu" type="checkbox" id="side-menu"/>
+        <label class="hamb" for="side-menu"><span class="hamb-line"></span></label>
+        <nav class="nav">
         <h1>
             X:if "@lang = 'he'" { X:attribute "align" { X:text{left} } }
             <a href="/">
-                <img src="/nginx.png" alt="nginx" />
+                <img src="/nginx.png" alt="nginx"/>
             </a>
         </h1>
         <div>
-        !! "document(concat($XML, '/menu.xml'))
+        <ul class="mobilemenu">
+            !! "document(concat($XML, '/menu.xml'))
                         /menus/menu[@lang = $lang]/item";
+        </ul>
         </div>
+        </nav>
     </div>
 
     <div id="content">
diff -r df1cf98cf8f5 -r c4018ab31dc5 xsls/menu.xsls
--- a/xsls/menu.xsls	Mon Jul 24 14:59:57 2023 +0100
+++ b/xsls/menu.xsls	Mon Jul 24 15:01:25 2023 +0100
@@ -13,6 +13,8 @@
       --     "menu/item[@href = $LINK]", etc.
       -->
 
+    <li>
+
     X:if "@href = $LINK" {
         X:if "$YEAR and @href='/'" {
             <a href="./"> news </a> <br/>
@@ -21,7 +23,6 @@
         }
 
     } else {
-
         <!--
           -- If a menu item has the switchlang attribute, then it will point
           -- to the same document in the specified language.
@@ -66,33 +67,34 @@
 
             X:if "@lang" { X:text { [} !{@lang} X:text {]}}
         }
-
-        <br/>
     }
+    </li>
 }
 
 
 X:template = "menu/item[@year]" {
     X:if "$YEAR or $LINK='/'" {
+        <li>
         X:if "$YEAR=@year" {
-            !{@year} <br/>
+            !{@year}
         } else {
             X:if "@href" { <a href="{@href}"> !{@year} </a> }
-            <br/>
         }
+        </li>
     }
 }
 
 
 X:template = "menu/item[starts-with(@href, 'http://') or starts-with(@href, 'https://')]" {
+    <li>
     <a href="{@href}"> !{ normalize-space(text()) } </a>
     X:if "@lang" { X:text { [} !{@lang} X:text {]}}
-    <br/>
+    </li>
 }
 
 
 X:template = "menu/item[not(@href) and not(@year)]" {
-    !{ normalize-space(text()) } <br/>
+    <li>!{ normalize-space(text()) } <br/></li>
 }
 
 }
diff -r df1cf98cf8f5 -r c4018ab31dc5 xsls/style.xsls
--- a/xsls/style.xsls	Mon Jul 24 14:59:57 2023 +0100
+++ b/xsls/style.xsls	Mon Jul 24 15:01:25 2023 +0100
@@ -7,6 +7,7 @@
 
 X:template style (lang) {
 
+    <meta name="viewport" content="width=device-width,initial-scale=1"></meta>
     <link rel="stylesheet" href="/css/style_{$lang}.css"></link>
 
 }

From l.crilly at f5.com  Mon Jul 24 14:26:41 2023
From: l.crilly at f5.com (Liam Crilly)
Date: Mon, 24 Jul 2023 14:26:41 +0000
Subject: [PATCH 3 of 3] Stylesheet style changes.
In-Reply-To: <BY3PR01MB6674B4CA03F27E73F9D3A0F7A502A@BY3PR01MB6674.prod.exchangelabs.com>
References: <BY3PR01MB667424E626632E5343465173A502A@BY3PR01MB6674.prod.exchangelabs.com>
 <BY3PR01MB6674D0B67BA5296084AC63A3A502A@BY3PR01MB6674.prod.exchangelabs.com>
 <BY3PR01MB6674B4CA03F27E73F9D3A0F7A502A@BY3PR01MB6674.prod.exchangelabs.com>
Message-ID: <BY3PR01MB6674C84893445FCA306897E4A502A@BY3PR01MB6674.prod.exchangelabs.com>

# HG changeset patch
# User Liam Crilly <liam.crilly at nginx.com>
# Date 1690207410 -3600
#      Mon Jul 24 15:03:30 2023 +0100
# Node ID 29929155db5ad60e9f9c201d74fb73889287848f
# Parent  c4018ab31dc52632f470298fcc7cece1fd8f57b9
Stylesheet style changes.

Optimized for readability, including support for dark mode.

diff -r c4018ab31dc5 -r 29929155db5a css/style_en.css
--- a/css/style_en.css	Mon Jul 24 15:01:25 2023 +0100
+++ b/css/style_en.css	Mon Jul 24 15:03:30 2023 +0100
@@ -9,13 +9,12 @@
 }
 
 #banner {
-    background: black;
-    color: #F2F2F2;
+    background: #000;
+    color: #f2f2f2;
     line-height: 1.2em;
     padding: .3em 0;
-    box-shadow: 0 5px 10px black;
+    box-shadow: 0 5px 10px #000;
 }
-
 #banner a {
     color: #00B140;
 }
@@ -24,7 +23,7 @@
     text-align: left;
     margin: 0 auto;
     min-width: 32em;
-    max-width: 64em;
+    max-width: 66em;
 }
 
 #menu {
@@ -46,9 +45,9 @@
     margin:0;
 }
 
- #content {
+#content {
     margin-right: 13.5em;
-    padding: 0 .2em 0 1.5em;
+    padding: 0 2.5em 0 1.5em;
 }
 
 h1 {
@@ -63,11 +62,23 @@
 }
 
 h2 {
-    text-align: center;
+    font-size: 2rem;
+    line-height: 3.25rem;
+    margin: 1rem 0 .5rem 0;
+}
+
+h4 {
+    font-size: 1.5rem;
+    margin: 2rem 0 .5rem 0;
+}
+/* Center field override */
+center h4 {
+    text-align: left!important;
 }
 
 p {
     text-align: justify;
+    margin-bottom: 0;
 }
 
 table.news p {
@@ -80,7 +91,7 @@
 
 table.news .date {
     text-align: right;
-    padding-right: 0.5em;
+    padding: 0.75rem 0.5rem 0 0;
     white-space: nowrap;
 }
 
@@ -94,10 +105,11 @@
 
 div.directive {
     background: #F2F2F2;
+    border-top: 2px solid #DDD;
     line-height: 1em;
     margin: 1em 0 1em -1em;
     padding: .7em .7em .7em 1em;
-    border-top: 2px solid #DDD;
+    width:100%;
 }
 
 div.directive th {
@@ -117,13 +129,19 @@
     margin: 0;
 }
 
+div.directive code {
+    background: none!important;
+    padding: 0!important;
+    border: none;
+}
+
 div.directive p {
     margin: .5em 0 0 .1em;
     font-size: .8em;
 }
 
 a.notrans {
-    color: gray;
+    color: #808080;
     text-decoration:none;
 }
 
@@ -143,7 +161,6 @@
 }
 
 li {
-    text-align: justify;
     padding: .5em 0 0 1px;
 }
 
@@ -156,7 +173,8 @@
 }
 
 dt {
-    margin: .5em 0;
+    margin: 1em 0 0 0!important;
+    font-size: 0.9rem;
 }
 
 .compact dt {
@@ -164,18 +182,18 @@
 }
 
 dd {
-    margin-left: 1.5em;
+    margin: 0.25rem 0 1.5rem 1rem;
     padding-left: 1px;
     text-align: justify;
 }
 
 td.list {
-    background: #F2F2F2;
+    background: #f2f2f2;
 }
 
 blockquote {
-    margin: 1em 0 1em 1em;
-    padding: .5em;
+    margin: 0.5rem 0 0 0;
+    padding: 0;
 }
 
 li blockquote, dd blockquote {
@@ -183,19 +201,49 @@
 }
 
 blockquote.note {
+    padding: 0.5rem;
     border: 1px dotted #999;
     line-height: 1.2em;
     text-align: justify;
 }
 
+pre {
+    border: 0;
+    display: block;
+    margin: 1em 0;
+    padding: 0.6em 1em;
+    overflow-x: auto;
+    line-height: 1.05rem;
+}
+
+code,.legacy{
+    font-size: 100%;
+    padding: 3px 7px;
+    overflow-x: scroll;
+    white-space: nowrap;
+    background: #eee;
+}
+
+pre code{
+    background: none;
+    padding: 0;
+    border-radius: 0;
+}
+
+blockquote pre {
+    background-color: #f8f8f8!important;
+    color: #222!important;
+    margin: 0;
+    border: 1px solid #ddd;
+}
+
+blockquote.terminal pre{
+    background-color: #444!important;
+    color: #fff!important;
+}
+
 blockquote.example {
     line-height: 1em;
-    border-left: 1px solid #BBB;
-}
-
-blockquote.example pre {
-    padding: 0;
-    margin: 0;
 }
 
 sup {
@@ -210,10 +258,10 @@
 
 .video iframe, .video object, .video embed {
     position: absolute;
-    top:0;
-    left:0;
-    width:100%;
-    height:100%;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
 }
 
 @media screen and (max-width:768px) {
@@ -337,3 +385,40 @@
         display: none;
     }
 }
+
+/* Dark Mode */
+ at media (prefers-color-scheme: dark) {
+	body {
+		background-color: #1c262d;
+        color:#eee;
+	}
+    h1,h2,h3,h4,h5,h6,p {
+        color: #eee;
+    }
+    a {
+        color: #55A4F1;
+        text-decoration: underline;
+    }
+    a:visited {
+        color: #8b79c0!important;
+    }
+    .nav {
+        background:var(--black);
+    }
+    #menu a {
+        color: #55A4F1;
+    }
+    blockquote pre {
+        color: #ddd!important;
+        background: #0f1b29!important;
+        border: 1px solid #132235;
+    }
+    div.directive {
+        background: #0c1622!important;
+        border-top: 2px solid #203c5e!important;
+    }
+    code {
+        background:#0c1622;
+        color: #ddd!important;
+    }
+}
diff -r c4018ab31dc5 -r 29929155db5a css/style_ru.css
--- a/css/style_ru.css	Mon Jul 24 15:01:25 2023 +0100
+++ b/css/style_ru.css	Mon Jul 24 15:03:30 2023 +0100
@@ -9,13 +9,12 @@
 }
 
 #banner {
-    background: black;
-    color: #F2F2F2;
+    background: #000;
+    color: #f2f2f2;
     line-height: 1.2em;
     padding: .3em 0;
-    box-shadow: 0 5px 10px black;
+    box-shadow: 0 5px 10px #000;
 }
-
 #banner a {
     color: #00B140;
 }
@@ -24,7 +23,7 @@
     text-align: left;
     margin: 0 auto;
     min-width: 32em;
-    max-width: 64em;
+    max-width: 66em;
 }
 
 #menu {
@@ -46,9 +45,9 @@
     margin:0;
 }
 
- #content {
+#content {
     margin-right: 13.5em;
-    padding: 0 .2em 0 1.5em;
+    padding: 0 2.5em 0 1.5em;
 }
 
 h1 {
@@ -63,11 +62,23 @@
 }
 
 h2 {
-    text-align: center;
+    font-size: 2rem;
+    line-height: 3.25rem;
+    margin: 1rem 0 .5rem 0;
+}
+
+h4 {
+    font-size: 1.5rem;
+    margin: 2rem 0 .5rem 0;
+}
+/* Center field override */
+center h4 {
+    text-align: left!important;
 }
 
 p {
     text-align: justify;
+    margin-bottom: 0;
 }
 
 table.news p {
@@ -80,7 +91,7 @@
 
 table.news .date {
     text-align: right;
-    padding-right: 0.5em;
+    padding: 0.75rem 0.5rem 0 0;
     white-space: nowrap;
 }
 
@@ -94,10 +105,11 @@
 
 div.directive {
     background: #F2F2F2;
+    border-top: 2px solid #DDD;
     line-height: 1em;
     margin: 1em 0 1em -1em;
     padding: .7em .7em .7em 1em;
-    border-top: 2px solid #DDD;
+    width:100%;
 }
 
 div.directive th {
@@ -117,13 +129,19 @@
     margin: 0;
 }
 
+div.directive code {
+    background: none!important;
+    padding: 0!important;
+    border: none;
+}
+
 div.directive p {
     margin: .5em 0 0 .1em;
     font-size: .8em;
 }
 
 a.notrans {
-    color: gray;
+    color: #808080;
     text-decoration:none;
 }
 
@@ -143,7 +161,6 @@
 }
 
 li {
-    text-align: justify;
     padding: .5em 0 0 1px;
 }
 
@@ -156,7 +173,8 @@
 }
 
 dt {
-    margin: .5em 0;
+    margin: 1em 0 0 0!important;
+    font-size: 0.9rem;
 }
 
 .compact dt {
@@ -164,18 +182,18 @@
 }
 
 dd {
-    margin-left: 1.5em;
+    margin: 0.25rem 0 1.5rem 1rem;
     padding-left: 1px;
     text-align: justify;
 }
 
 td.list {
-    background: #F2F2F2;
+    background: #f2f2f2;
 }
 
 blockquote {
-    margin: 1em 0 1em 1em;
-    padding: .5em;
+    margin: 0.5rem 0 0 0;
+    padding: 0;
 }
 
 li blockquote, dd blockquote {
@@ -183,19 +201,49 @@
 }
 
 blockquote.note {
+    padding: 0.5rem;
     border: 1px dotted #999;
     line-height: 1.2em;
     text-align: justify;
 }
 
+pre {
+    border: 0;
+    display: block;
+    margin: 1em 0;
+    padding: 0.6em 1em;
+    overflow-x: auto;
+    line-height: 1.05rem;
+}
+
+code,.legacy{
+    font-size: 100%;
+    padding: 3px 7px;
+    overflow-x: scroll;
+    white-space: nowrap;
+    background: #eee;
+}
+
+pre code{
+    background: none;
+    padding: 0;
+    border-radius: 0;
+}
+
+blockquote pre {
+    background-color: #f8f8f8!important;
+    color: #222!important;
+    margin: 0;
+    border: 1px solid #ddd;
+}
+
+blockquote.terminal pre{
+    background-color: #444!important;
+    color: #fff!important;
+}
+
 blockquote.example {
     line-height: 1em;
-    border-left: 1px solid #BBB;
-}
-
-blockquote.example pre {
-    padding: 0;
-    margin: 0;
 }
 
 sup {
@@ -210,10 +258,10 @@
 
 .video iframe, .video object, .video embed {
     position: absolute;
-    top:0;
-    left:0;
-    width:100%;
-    height:100%;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
 }
 
 @media screen and (max-width:768px) {
@@ -337,3 +385,40 @@
         display: none;
     }
 }
+
+/* Dark Mode */
+ at media (prefers-color-scheme: dark) {
+	body {
+		background-color: #1c262d;
+        color:#eee;
+	}
+    h1,h2,h3,h4,h5,h6,p {
+        color: #eee;
+    }
+    a {
+        color: #55A4F1;
+        text-decoration: underline;
+    }
+    a:visited {
+        color: #8b79c0!important;
+    }
+    .nav {
+        background:var(--black);
+    }
+    #menu a {
+        color: #55A4F1;
+    }
+    blockquote pre {
+        color: #ddd!important;
+        background: #0f1b29!important;
+        border: 1px solid #132235;
+    }
+    div.directive {
+        background: #0c1622!important;
+        border-top: 2px solid #203c5e!important;
+    }
+    code {
+        background:#0c1622;
+        color: #ddd!important;
+    }
+}

From gmm at csdoc.com  Mon Jul 24 15:17:22 2023
From: gmm at csdoc.com (Gena Makhomed)
Date: Mon, 24 Jul 2023 18:17:22 +0300
Subject: [PATCH] Contrib: vim syntax, update core and 3rd party module
 directives.
In-Reply-To: <ZL3NZ3xrWy4B/NTn@mdounin.ru>
References: <5f76c221-445a-ff97-f9b2-04430b0902d1@csdoc.com>
 <ZLlhs6M2sHZWdp3k@mdounin.ru>
 <7cd58e2b-b247-6230-76a9-21bf1cdeca1d@csdoc.com>
 <ZL3NZ3xrWy4B/NTn@mdounin.ru>
Message-ID: <cf3a4420-38e8-22ee-e79a-0834dfe31320@csdoc.com>

# HG changeset patch
# User Gena Makhomed <gmm at csdoc.com>
# Date 1690211081 -10800
#      Mon Jul 24 18:04:41 2023 +0300
# Node ID 904c99bede1770d92566b56939c5b6ec85f05b55
# Parent  c209dc4eed1737261c97a044d23f5fb9afbd9d45
Contrib: vim syntax, update core and 3rd party module directives.

List of 3rd party modules github repositories was obtained from
https://github.com/freebsd/freebsd-ports/blob/main/www/nginx-devel/Makefile.extmod

diff -r c209dc4eed17 -r 904c99bede17 contrib/vim/syntax/nginx.vim
--- a/contrib/vim/syntax/nginx.vim	Wed Jul 19 05:09:23 2023 +0300
+++ b/contrib/vim/syntax/nginx.vim	Mon Jul 24 18:04:41 2023 +0300
@@ -65,12 +65,12 @@
      \ contained
      \ nextgroup=@ngxListenParams skipwhite skipempty
  syn keyword ngxListenOptions contained
-    \ default_server ssl http2 proxy_protocol
+    \ default_server ssl quic proxy_protocol
      \ setfib fastopen backlog rcvbuf sndbuf accept_filter deferred bind
      \ ipv6only reuseport so_keepalive
      \ nextgroup=@ngxListenParams skipwhite skipempty
  syn keyword ngxListenOptionsDeprecated contained
-    \ spdy
+    \ http2
      \ nextgroup=@ngxListenParams skipwhite skipempty
  syn cluster ngxListenParams
      \ contains=ngxListenParam,ngxListenString,ngxListenComment
@@ -90,7 +90,6 @@
  syn keyword ngxDirectiveBlock contained geo
  syn keyword ngxDirectiveBlock contained map
  syn keyword ngxDirectiveBlock contained split_clients
-syn keyword ngxDirectiveBlock contained match

  syn keyword ngxDirectiveImportant contained include
  syn keyword ngxDirectiveImportant contained root
@@ -113,7 +112,6 @@

  syn keyword ngxDirectiveDeprecated contained proxy_downstream_buffer
  syn keyword ngxDirectiveDeprecated contained proxy_upstream_buffer
-syn keyword ngxDirectiveDeprecated contained ssl
  syn keyword ngxDirectiveDeprecated contained http2_idle_timeout
  syn keyword ngxDirectiveDeprecated contained http2_max_field_size
  syn keyword ngxDirectiveDeprecated contained http2_max_header_size
@@ -136,7 +134,6 @@
  syn keyword ngxDirective contained allow
  syn keyword ngxDirective contained ancient_browser
  syn keyword ngxDirective contained ancient_browser_value
-syn keyword ngxDirective contained api
  syn keyword ngxDirective contained auth_basic
  syn keyword ngxDirective contained auth_basic_user_file
  syn keyword ngxDirective contained auth_delay
@@ -144,15 +141,6 @@
  syn keyword ngxDirective contained auth_http_header
  syn keyword ngxDirective contained auth_http_pass_client_cert
  syn keyword ngxDirective contained auth_http_timeout
-syn keyword ngxDirective contained auth_jwt
-syn keyword ngxDirective contained auth_jwt_claim_set
-syn keyword ngxDirective contained auth_jwt_header_set
-syn keyword ngxDirective contained auth_jwt_key_cache
-syn keyword ngxDirective contained auth_jwt_key_file
-syn keyword ngxDirective contained auth_jwt_key_request
-syn keyword ngxDirective contained auth_jwt_leeway
-syn keyword ngxDirective contained auth_jwt_require
-syn keyword ngxDirective contained auth_jwt_type
  syn keyword ngxDirective contained auth_request
  syn keyword ngxDirective contained auth_request_set
  syn keyword ngxDirective contained autoindex
@@ -193,8 +181,6 @@
  syn keyword ngxDirective contained etag
  syn keyword ngxDirective contained eventport_events
  syn keyword ngxDirective contained expires
-syn keyword ngxDirective contained f4f
-syn keyword ngxDirective contained f4f_buffer_size
  syn keyword ngxDirective contained fastcgi_bind
  syn keyword ngxDirective contained fastcgi_buffer_size
  syn keyword ngxDirective contained fastcgi_buffering
@@ -211,7 +197,6 @@
  syn keyword ngxDirective contained fastcgi_cache_methods
  syn keyword ngxDirective contained fastcgi_cache_min_uses
  syn keyword ngxDirective contained fastcgi_cache_path
-syn keyword ngxDirective contained fastcgi_cache_purge
  syn keyword ngxDirective contained fastcgi_cache_revalidate
  syn keyword ngxDirective contained fastcgi_cache_use_stale
  syn keyword ngxDirective contained fastcgi_cache_valid
@@ -295,14 +280,7 @@
  syn keyword ngxDirective contained gzip_vary
  syn keyword ngxDirective contained gzip_window
  syn keyword ngxDirective contained hash
-syn keyword ngxDirective contained health_check
-syn keyword ngxDirective contained health_check_timeout
-syn keyword ngxDirective contained hls
-syn keyword ngxDirective contained hls_buffers
-syn keyword ngxDirective contained hls_forward_args
-syn keyword ngxDirective contained hls_fragment
-syn keyword ngxDirective contained hls_mp4_buffer_size
-syn keyword ngxDirective contained hls_mp4_max_buffer_size
+syn keyword ngxDirective contained http2
  syn keyword ngxDirective contained http2_body_preread_size
  syn keyword ngxDirective contained http2_chunk_size
  syn keyword ngxDirective contained http2_max_concurrent_pushes
@@ -312,6 +290,10 @@
  syn keyword ngxDirective contained http2_push_preload
  syn keyword ngxDirective contained http2_recv_buffer_size
  syn keyword ngxDirective contained http2_streams_index_size
+syn keyword ngxDirective contained http3
+syn keyword ngxDirective contained http3_hq
+syn keyword ngxDirective contained http3_max_concurrent_streams
+syn keyword ngxDirective contained http3_stream_buffer_size
  syn keyword ngxDirective contained if_modified_since
  syn keyword ngxDirective contained ignore_invalid_headers
  syn keyword ngxDirective contained image_filter
@@ -342,21 +324,20 @@
  syn keyword ngxDirective contained js_header_filter
  syn keyword ngxDirective contained js_import
  syn keyword ngxDirective contained js_path
+syn keyword ngxDirective contained js_preload_object
  syn keyword ngxDirective contained js_preread
  syn keyword ngxDirective contained js_set
+syn keyword ngxDirective contained js_shared_dict_zone
  syn keyword ngxDirective contained js_var
  syn keyword ngxDirective contained keepalive
  syn keyword ngxDirective contained keepalive_disable
  syn keyword ngxDirective contained keepalive_requests
  syn keyword ngxDirective contained keepalive_time
  syn keyword ngxDirective contained keepalive_timeout
-syn keyword ngxDirective contained keyval
-syn keyword ngxDirective contained keyval_zone
  syn keyword ngxDirective contained kqueue_changes
  syn keyword ngxDirective contained kqueue_events
  syn keyword ngxDirective contained large_client_header_buffers
  syn keyword ngxDirective contained least_conn
-syn keyword ngxDirective contained least_time
  syn keyword ngxDirective contained limit_conn
  syn keyword ngxDirective contained limit_conn_dry_run
  syn keyword ngxDirective contained limit_conn_log_level
@@ -400,14 +381,11 @@
  syn keyword ngxDirective contained modern_browser_value
  syn keyword ngxDirective contained mp4
  syn keyword ngxDirective contained mp4_buffer_size
-syn keyword ngxDirective contained mp4_limit_rate
-syn keyword ngxDirective contained mp4_limit_rate_after
  syn keyword ngxDirective contained mp4_max_buffer_size
  syn keyword ngxDirective contained mp4_start_key_frame
  syn keyword ngxDirective contained msie_padding
  syn keyword ngxDirective contained msie_refresh
  syn keyword ngxDirective contained multi_accept
-syn keyword ngxDirective contained ntlm
  syn keyword ngxDirective contained open_file_cache
  syn keyword ngxDirective contained open_file_cache_errors
  syn keyword ngxDirective contained open_file_cache_events
@@ -450,7 +428,6 @@
  syn keyword ngxDirective contained proxy_cache_methods
  syn keyword ngxDirective contained proxy_cache_min_uses
  syn keyword ngxDirective contained proxy_cache_path
-syn keyword ngxDirective contained proxy_cache_purge
  syn keyword ngxDirective contained proxy_cache_revalidate
  syn keyword ngxDirective contained proxy_cache_use_stale
  syn keyword ngxDirective contained proxy_cache_valid
@@ -488,7 +465,6 @@
  syn keyword ngxDirective contained proxy_responses
  syn keyword ngxDirective contained proxy_send_lowat
  syn keyword ngxDirective contained proxy_send_timeout
-syn keyword ngxDirective contained proxy_session_drop
  syn keyword ngxDirective contained proxy_set_body
  syn keyword ngxDirective contained proxy_set_header
  syn keyword ngxDirective contained proxy_smtp_auth
@@ -513,7 +489,11 @@
  syn keyword ngxDirective contained proxy_temp_path
  syn keyword ngxDirective contained proxy_timeout
  syn keyword ngxDirective contained proxy_upload_rate
-syn keyword ngxDirective contained queue
+syn keyword ngxDirective contained quic_active_connection_id_limit
+syn keyword ngxDirective contained quic_bpf
+syn keyword ngxDirective contained quic_gso
+syn keyword ngxDirective contained quic_host_key
+syn keyword ngxDirective contained quic_retry
  syn keyword ngxDirective contained random
  syn keyword ngxDirective contained random_index
  syn keyword ngxDirective contained read_ahead
@@ -544,7 +524,6 @@
  syn keyword ngxDirective contained scgi_cache_methods
  syn keyword ngxDirective contained scgi_cache_min_uses
  syn keyword ngxDirective contained scgi_cache_path
-syn keyword ngxDirective contained scgi_cache_purge
  syn keyword ngxDirective contained scgi_cache_revalidate
  syn keyword ngxDirective contained scgi_cache_use_stale
  syn keyword ngxDirective contained scgi_cache_valid
@@ -583,9 +562,6 @@
  syn keyword ngxDirective contained server_names_hash_bucket_size
  syn keyword ngxDirective contained server_names_hash_max_size
  syn keyword ngxDirective contained server_tokens
-syn keyword ngxDirective contained session_log
-syn keyword ngxDirective contained session_log_format
-syn keyword ngxDirective contained session_log_zone
  syn keyword ngxDirective contained set_real_ip_from
  syn keyword ngxDirective contained slice
  syn keyword ngxDirective contained smtp_auth
@@ -633,11 +609,6 @@
  syn keyword ngxDirective contained ssl_verify_client
  syn keyword ngxDirective contained ssl_verify_depth
  syn keyword ngxDirective contained starttls
-syn keyword ngxDirective contained state
-syn keyword ngxDirective contained status
-syn keyword ngxDirective contained status_format
-syn keyword ngxDirective contained status_zone
-syn keyword ngxDirective contained sticky
  syn keyword ngxDirective contained stub_status
  syn keyword ngxDirective contained sub_filter
  syn keyword ngxDirective contained sub_filter_last_modified
@@ -680,7 +651,6 @@
  syn keyword ngxDirective contained uwsgi_cache_methods
  syn keyword ngxDirective contained uwsgi_cache_min_uses
  syn keyword ngxDirective contained uwsgi_cache_path
-syn keyword ngxDirective contained uwsgi_cache_purge
  syn keyword ngxDirective contained uwsgi_cache_revalidate
  syn keyword ngxDirective contained uwsgi_cache_use_stale
  syn keyword ngxDirective contained uwsgi_cache_valid
@@ -744,6 +714,62 @@
  syn keyword ngxDirective contained xslt_stylesheet
  syn keyword ngxDirective contained xslt_types
  syn keyword ngxDirective contained zone
+
+" nginx-plus commercial extensions directives
+
+syn keyword ngxDirectiveBlock contained match
+syn keyword ngxDirectiveBlock contained otel_exporter
+
+syn keyword ngxDirective contained api
+syn keyword ngxDirective contained auth_jwt
+syn keyword ngxDirective contained auth_jwt_claim_set
+syn keyword ngxDirective contained auth_jwt_header_set
+syn keyword ngxDirective contained auth_jwt_key_cache
+syn keyword ngxDirective contained auth_jwt_key_file
+syn keyword ngxDirective contained auth_jwt_key_request
+syn keyword ngxDirective contained auth_jwt_leeway
+syn keyword ngxDirective contained auth_jwt_require
+syn keyword ngxDirective contained auth_jwt_type
+syn keyword ngxDirective contained f4f
+syn keyword ngxDirective contained f4f_buffer_size
+syn keyword ngxDirective contained fastcgi_cache_purge
+syn keyword ngxDirective contained health_check
+syn keyword ngxDirective contained health_check_timeout
+syn keyword ngxDirective contained hls
+syn keyword ngxDirective contained hls_buffers
+syn keyword ngxDirective contained hls_forward_args
+syn keyword ngxDirective contained hls_fragment
+syn keyword ngxDirective contained hls_mp4_buffer_size
+syn keyword ngxDirective contained hls_mp4_max_buffer_size
+syn keyword ngxDirective contained internal_redirect
+syn keyword ngxDirective contained keyval
+syn keyword ngxDirective contained keyval_zone
+syn keyword ngxDirective contained least_time
+syn keyword ngxDirective contained mp4_limit_rate
+syn keyword ngxDirective contained mp4_limit_rate_after
+syn keyword ngxDirective contained mqtt
+syn keyword ngxDirective contained mqtt_preread
+syn keyword ngxDirective contained mqtt_rewrite_buffer_size
+syn keyword ngxDirective contained mqtt_set_connect
+syn keyword ngxDirective contained ntlm
+syn keyword ngxDirective contained otel_service_name
+syn keyword ngxDirective contained otel_span_attr
+syn keyword ngxDirective contained otel_span_name
+syn keyword ngxDirective contained otel_trace
+syn keyword ngxDirective contained otel_trace_context
+syn keyword ngxDirective contained proxy_cache_purge
+syn keyword ngxDirective contained proxy_session_drop
+syn keyword ngxDirective contained queue
+syn keyword ngxDirective contained scgi_cache_purge
+syn keyword ngxDirective contained session_log
+syn keyword ngxDirective contained session_log_format
+syn keyword ngxDirective contained session_log_zone
+syn keyword ngxDirective contained state
+syn keyword ngxDirective contained status
+syn keyword ngxDirective contained status_format
+syn keyword ngxDirective contained status_zone
+syn keyword ngxDirective contained sticky
+syn keyword ngxDirective contained uwsgi_cache_purge
  syn keyword ngxDirective contained zone_sync
  syn keyword ngxDirective contained zone_sync_buffers
  syn keyword ngxDirective contained zone_sync_connect_retry_interval
@@ -766,7 +792,6 @@
  syn keyword ngxDirective contained zone_sync_ssl_verify_depth
  syn keyword ngxDirective contained zone_sync_timeout

-
  " 3rd party modules list taken from
  " 
https://github.com/freebsd/freebsd-ports/blob/main/www/nginx-devel/Makefile.extmod
  " 
----------------------------------------------------------------------------------
@@ -837,52 +862,6 @@
  " https://github.com/torden/ngx_cache_purge
  syn keyword ngxDirectiveThirdParty contained cache_purge_response_type

-" https://github.com/nginx-clojure/nginx-clojure
-syn keyword ngxDirectiveThirdParty contained access_handler_code
-syn keyword ngxDirectiveThirdParty contained access_handler_name
-syn keyword ngxDirectiveThirdParty contained access_handler_property
-syn keyword ngxDirectiveThirdParty contained access_handler_type
-syn keyword ngxDirectiveThirdParty contained always_read_body
-syn keyword ngxDirectiveThirdParty contained auto_upgrade_ws
-syn keyword ngxDirectiveThirdParty contained body_filter_code
-syn keyword ngxDirectiveThirdParty contained body_filter_name
-syn keyword ngxDirectiveThirdParty contained body_filter_property
-syn keyword ngxDirectiveThirdParty contained body_filter_type
-syn keyword ngxDirectiveThirdParty contained content_handler_code
-syn keyword ngxDirectiveThirdParty contained content_handler_name
-syn keyword ngxDirectiveThirdParty contained content_handler_property
-syn keyword ngxDirectiveThirdParty contained content_handler_type
-syn keyword ngxDirectiveThirdParty contained handler_code
-syn keyword ngxDirectiveThirdParty contained handler_name
-syn keyword ngxDirectiveThirdParty contained handler_type
-syn keyword ngxDirectiveThirdParty contained handlers_lazy_init
-syn keyword ngxDirectiveThirdParty contained header_filter_code
-syn keyword ngxDirectiveThirdParty contained header_filter_name
-syn keyword ngxDirectiveThirdParty contained header_filter_property
-syn keyword ngxDirectiveThirdParty contained header_filter_type
-syn keyword ngxDirectiveThirdParty contained jvm_classpath
-syn keyword ngxDirectiveThirdParty contained jvm_classpath_check
-syn keyword ngxDirectiveThirdParty contained jvm_exit_handler_code
-syn keyword ngxDirectiveThirdParty contained jvm_exit_handler_name
-syn keyword ngxDirectiveThirdParty contained jvm_handler_type
-syn keyword ngxDirectiveThirdParty contained jvm_init_handler_code
-syn keyword ngxDirectiveThirdParty contained jvm_init_handler_name
-syn keyword ngxDirectiveThirdParty contained jvm_options
-syn keyword ngxDirectiveThirdParty contained jvm_path
-syn keyword ngxDirectiveThirdParty contained jvm_var
-syn keyword ngxDirectiveThirdParty contained jvm_workers
-syn keyword ngxDirectiveThirdParty contained log_handler_code
-syn keyword ngxDirectiveThirdParty contained log_handler_name
-syn keyword ngxDirectiveThirdParty contained log_handler_property
-syn keyword ngxDirectiveThirdParty contained log_handler_type
-syn keyword ngxDirectiveThirdParty contained max_balanced_tcp_connections
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_code
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_name
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_property
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_type
-syn keyword ngxDirectiveThirdParty contained shared_map
-syn keyword ngxDirectiveThirdParty contained write_page_size
-
  " https://github.com/AirisX/nginx_cookie_flag_module
  syn keyword ngxDirectiveThirdParty contained set_cookie_flag

@@ -932,29 +911,6 @@
  syn keyword ngxDirectiveThirdParty contained dynamic_state_file
  syn keyword ngxDirectiveThirdParty contained dynamic_upstream

-" https://github.com/ZigzagAK/ngx_dynamic_healthcheck
-syn keyword ngxDirectiveThirdParty contained check
-syn keyword ngxDirectiveThirdParty contained check_disable_host
-syn keyword ngxDirectiveThirdParty contained check_exclude_host
-syn keyword ngxDirectiveThirdParty contained check_persistent
-syn keyword ngxDirectiveThirdParty contained check_request_body
-syn keyword ngxDirectiveThirdParty contained check_request_headers
-syn keyword ngxDirectiveThirdParty contained check_request_uri
-syn keyword ngxDirectiveThirdParty contained check_response_body
-syn keyword ngxDirectiveThirdParty contained check_response_codes
-syn keyword ngxDirectiveThirdParty contained healthcheck
-syn keyword ngxDirectiveThirdParty contained healthcheck_buffer_size
-syn keyword ngxDirectiveThirdParty contained healthcheck_disable_host
-syn keyword ngxDirectiveThirdParty contained healthcheck_get
-syn keyword ngxDirectiveThirdParty contained healthcheck_persistent
-syn keyword ngxDirectiveThirdParty contained healthcheck_request_body
-syn keyword ngxDirectiveThirdParty contained healthcheck_request_headers
-syn keyword ngxDirectiveThirdParty contained healthcheck_request_uri
-syn keyword ngxDirectiveThirdParty contained healthcheck_response_body
-syn keyword ngxDirectiveThirdParty contained healthcheck_response_codes
-syn keyword ngxDirectiveThirdParty contained healthcheck_status
-syn keyword ngxDirectiveThirdParty contained healthcheck_update
-
  " https://github.com/openresty/encrypted-session-nginx-module
  syn keyword ngxDirectiveThirdParty contained encrypted_session_expires
  syn keyword ngxDirectiveThirdParty contained encrypted_session_iv
@@ -1004,6 +960,7 @@
  syn keyword ngxDirectiveThirdParty contained auth_gss_realm
  syn keyword ngxDirectiveThirdParty contained auth_gss_service_ccache
  syn keyword ngxDirectiveThirdParty contained auth_gss_service_name
+syn keyword ngxDirectiveThirdParty contained auth_gss_zone_name

  " https://github.com/kvspb/nginx-auth-ldap
  syn keyword ngxDirectiveThirdParty contained auth_ldap
@@ -1033,6 +990,7 @@

  " https://github.com/aperezdc/ngx-fancyindex
  syn keyword ngxDirectiveThirdParty contained fancyindex
+syn keyword ngxDirectiveThirdParty contained fancyindex_case_sensitive
  syn keyword ngxDirectiveThirdParty contained fancyindex_css_href
  syn keyword ngxDirectiveThirdParty contained fancyindex_default_sort
  syn keyword ngxDirectiveThirdParty contained fancyindex_directories_first
@@ -1121,6 +1079,7 @@
  syn keyword ngxDirectiveThirdParty contained nchan_pubsub
  syn keyword ngxDirectiveThirdParty contained nchan_pubsub_channel_id
  syn keyword ngxDirectiveThirdParty contained nchan_pubsub_location
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_accurate_subscriber_count
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_cluster_check_interval
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_cluster_check_interval_backoff
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_cluster_check_interval_jitter
@@ -1138,6 +1097,11 @@
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_discovered_ip_range_blacklist
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_fakesub_timer_interval
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_cache_timeout
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_keepalive_backoff
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_keepalive_jitter
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_keepalive_max
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_keepalive_min
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_idle_channel_keepalive_safety_margin
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_load_scripts_unconditionally
  syn keyword ngxDirectiveThirdParty contained nchan_redis_namespace
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_node_connect_timeout
@@ -1173,6 +1137,9 @@
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_tls_trusted_certificate
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_tls_trusted_certificate_path
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_tls_verify_certificate
+syn keyword ngxDirectiveThirdParty contained nchan_redis_upstream_stats
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_upstream_stats_disconnected_timeout
+syn keyword ngxDirectiveThirdParty contained 
nchan_redis_upstream_stats_enabled
  syn keyword ngxDirectiveThirdParty contained nchan_redis_url
  syn keyword ngxDirectiveThirdParty contained nchan_redis_username
  syn keyword ngxDirectiveThirdParty contained 
nchan_redis_wait_after_connecting
@@ -1323,6 +1290,7 @@
  syn keyword ngxDirectiveThirdParty contained upload_progress_template

  " https://github.com/yaoweibin/nginx_upstream_check_module
+syn keyword ngxDirectiveThirdParty contained check
  syn keyword ngxDirectiveThirdParty contained check_fastcgi_param
  syn keyword ngxDirectiveThirdParty contained check_http_expect_alive
  syn keyword ngxDirectiveThirdParty contained check_http_send
@@ -1335,6 +1303,7 @@
  syn keyword ngxDirectiveThirdParty contained upstream_fair_shm_size

  " https://github.com/ayty-adrianomartins/nginx-sticky-module-ng
+syn keyword ngxDirectiveThirdParty contained sticky_hide_cookie
  syn keyword ngxDirectiveThirdParty contained sticky_no_fallback

  " https://github.com/Novetta/nginx-video-thumbextractor-module
@@ -1421,6 +1390,8 @@
  syn keyword ngxDirectiveThirdParty contained lua_socket_read_timeout
  syn keyword ngxDirectiveThirdParty contained lua_socket_send_lowat
  syn keyword ngxDirectiveThirdParty contained lua_socket_send_timeout
+syn keyword ngxDirectiveThirdParty contained lua_ssl_certificate
+syn keyword ngxDirectiveThirdParty contained lua_ssl_certificate_key
  syn keyword ngxDirectiveThirdParty contained lua_ssl_ciphers
  syn keyword ngxDirectiveThirdParty contained lua_ssl_conf_command
  syn keyword ngxDirectiveThirdParty contained lua_ssl_crl
@@ -1834,16 +1805,6 @@
  syn keyword ngxDirectiveThirdParty contained slowfs_cache_valid
  syn keyword ngxDirectiveThirdParty contained slowfs_temp_path

-" https://github.com/kawakibi/ngx_small_light
-syn keyword ngxDirectiveThirdParty contained small_light
-syn keyword ngxDirectiveThirdParty contained small_light_buffer
-syn keyword ngxDirectiveThirdParty contained small_light_getparam_mode
-syn keyword ngxDirectiveThirdParty contained small_light_imlib2_temp_dir
-syn keyword ngxDirectiveThirdParty contained small_light_material_dir
-syn keyword ngxDirectiveThirdParty contained small_light_pattern_define
-syn keyword ngxDirectiveThirdParty contained small_light_radius_max
-syn keyword ngxDirectiveThirdParty contained small_light_sigma_max
-
  " https://github.com/openresty/srcache-nginx-module
  syn keyword ngxDirectiveThirdParty contained srcache_buffer
  syn keyword ngxDirectiveThirdParty contained srcache_default_expire
@@ -1980,6 +1941,14 @@
  syn keyword ngxDirectiveThirdParty contained websockify_read_timeout
  syn keyword ngxDirectiveThirdParty contained websockify_send_timeout

+" https://github.com/vozlt/nginx-module-sts
+syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status
+syn keyword ngxDirectiveThirdParty contained 
stream_server_traffic_status_average_method
+syn keyword ngxDirectiveThirdParty contained 
stream_server_traffic_status_display
+syn keyword ngxDirectiveThirdParty contained 
stream_server_traffic_status_display_format
+syn keyword ngxDirectiveThirdParty contained 
stream_server_traffic_status_display_jsonp
+syn keyword ngxDirectiveThirdParty contained 
stream_server_traffic_status_zone
+
  " highlight

  hi def link ngxComment Comment

From mdounin at mdounin.ru  Mon Jul 24 18:36:03 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 24 Jul 2023 21:36:03 +0300
Subject: [PATCH] Contrib: vim syntax, update core and 3rd party module
 directives.
In-Reply-To: <cf3a4420-38e8-22ee-e79a-0834dfe31320@csdoc.com>
References: <5f76c221-445a-ff97-f9b2-04430b0902d1@csdoc.com>
 <ZLlhs6M2sHZWdp3k@mdounin.ru>
 <7cd58e2b-b247-6230-76a9-21bf1cdeca1d@csdoc.com>
 <ZL3NZ3xrWy4B/NTn@mdounin.ru>
 <cf3a4420-38e8-22ee-e79a-0834dfe31320@csdoc.com>
Message-ID: <ZL7Ek46p/K1u6+gm@mdounin.ru>

Hello!

On Mon, Jul 24, 2023 at 06:17:22PM +0300, Gena Makhomed wrote:

> # HG changeset patch
> # User Gena Makhomed <gmm at csdoc.com>
> # Date 1690211081 -10800
> #      Mon Jul 24 18:04:41 2023 +0300
> # Node ID 904c99bede1770d92566b56939c5b6ec85f05b55
> # Parent  c209dc4eed1737261c97a044d23f5fb9afbd9d45
> Contrib: vim syntax, update core and 3rd party module directives.
> 
> List of 3rd party modules github repositories was obtained from
> https://github.com/freebsd/freebsd-ports/blob/main/www/nginx-devel/Makefile.extmod

Pushed to http://mdounin.ru/hg/nginx, thanks!

[...]

-- 
Maxim Dounin
http://mdounin.ru/

From marcus.lima at azion.com  Mon Jul 24 19:03:33 2023
From: marcus.lima at azion.com (Marcus Lima)
Date: Mon, 24 Jul 2023 16:03:33 -0300
Subject: Question about QUIC and Reload
Message-ID: <CAKbqR297DnG1xxvmxY4cDRYy91-BnVhswXGT3YQ2E7U4HFA3cA@mail.gmail.com>

Hi everyone!

  I have had behaviour where workers while reloading, continue to receive
connection attempts, which is noticeable by the logs indicating that QUIC
Initial (ClientHello) packets have been submitted. When this occurs, the
connection client does not receive any responses until the timeout occurs.

  I expected that when the worker entered the "retire" state, it would not
receive the ClientHello. Would this behaviour be expected?
_______________________________________________________
2023/07/20 14:43:00 [notice] 7954#0: gracefully shutting down
2023/07/20 14:43:00 [debug] 7954#0: setproctitle: "nginx: worker process is
shutting down"

2023/07/20 14:43:20 [debug] 7954#0: *70058 quic recvmsg: *omitted* fd:28
n:1252
2023/07/20 14:43:20 [debug] 7954#0: *70058 quic packet done rc:-1
level:init decr:0 pn:0 perr:0
2023/07/20 14:43:20 [debug] 7954#0: *70058 quic packet rejected rc:-1,
cleanup connection

2023/07/20 14:43:33 [notice] 302#0: signal 17 (SIGCHLD) received from 7954

Cheers,

-- 
*Marcus Lima*,
Senior Software Security Engineer

AZION | Move to the Edge ®

Office: +55 51 3083 8101

Any information in this e-mail and attachments may be confidential and
privileged, protected by legal confidentiality. The use of this document
requires authorization by the issuer, subject to penalties.
Quaisquer informações contidas neste e-mail e anexos podem ser
confidenciais e privilegiadas, protegidas por sigilo legal. Qualquer forma
de utilização deste documento depende de autorização do emissor, sujeito às
penalidades cabíveis.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20230724/82a08eda/attachment-0001.htm>

From mdounin at mdounin.ru  Mon Jul 24 23:10:12 2023
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Tue, 25 Jul 2023 02:10:12 +0300
Subject: [PATCH 1 of 2] SSL: provided "nginx" appname when loading OpenSSL
 configs
Message-ID: <b79ef48b91e45dba4bf8.1690240212@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1687300195 -10800
#      Wed Jun 21 01:29:55 2023 +0300
# Node ID b79ef48b91e45dba4bf850be6b2a2cc3b8834f5d
# Parent  904c99bede1770d92566b56939c5b6ec85f05b55
SSL: provided "nginx" appname when loading OpenSSL configs.

Following OpenSSL 0.9.8f, OpenSSL tries to load application-specific
configuration section first, and then falls back to the "openssl_conf"
default section if application-specific section is not found, by using
CONF_modules_load_file(CONF_MFLAGS_DEFAULT_SECTION).  Therefore this
change is not expected to introduce any compatibility issues with existing
configurations.  It does, however, makes it easier to configure specific
OpenSSL settings for nginx in system-wide OpenSSL configuration
(ticket #2449).

Instead of checking OPENSSL_VERSION_NUMBER when using the OPENSSL_init_ssl()
interface, the code now tests for OPENSSL_INIT_LOAD_CONFIG to be defined and
true, and also explicitly excludes LibreSSL.  This ensures that this interface
is not used with BoringSSL and LibreSSL, which do not provide additional
library initialization settings, notably the OPENSSL_INIT_set_config_appname()
call.

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -140,13 +140,31 @@ int  ngx_ssl_stapling_index;
 ngx_int_t
 ngx_ssl_init(ngx_log_t *log)
 {
-#if OPENSSL_VERSION_NUMBER >= 0x10100003L
-
-    if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL) == 0) {
+#if (OPENSSL_INIT_LOAD_CONFIG && !defined LIBRESSL_VERSION_NUMBER)
+
+    OPENSSL_INIT_SETTINGS  *init;
+
+    init = OPENSSL_INIT_new();
+    if (init == NULL) {
+        ngx_ssl_error(NGX_LOG_ALERT, log, 0, "OPENSSL_INIT_new() failed");
+        return NGX_ERROR;
+    }
+
+#ifndef OPENSSL_NO_STDIO
+    if (OPENSSL_INIT_set_config_appname(init, "nginx") == 0) {
+        ngx_ssl_error(NGX_LOG_ALERT, log, 0,
+                      "OPENSSL_INIT_set_config_appname() failed");
+        return NGX_ERROR;
+    }
+#endif
+
+    if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, init) == 0) {
         ngx_ssl_error(NGX_LOG_ALERT, log, 0, "OPENSSL_init_ssl() failed");
         return NGX_ERROR;
     }
 
+    OPENSSL_INIT_free(init);
+
     /*
      * OPENSSL_init_ssl() may leave errors in the error queue
      * while returning success
@@ -156,7 +174,7 @@ ngx_ssl_init(ngx_log_t *log)
 
 #else
 
-    OPENSSL_config(NULL);
+    OPENSSL_config("nginx");
 
     SSL_library_init();
     SSL_load_error_strings();

From mdounin at mdounin.ru  Mon Jul 24 23:10:13 2023
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Tue, 25 Jul 2023 02:10:13 +0300
Subject: [PATCH 2 of 2] SSL: avoid using OpenSSL config in build directory
 (ticket #2404)
In-Reply-To: <b79ef48b91e45dba4bf8.1690240212@vm-bsd.mdounin.ru>
References: <b79ef48b91e45dba4bf8.1690240212@vm-bsd.mdounin.ru>
Message-ID: <bd2cc76ebe2367dc303e.1690240213@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1687300193 -10800
#      Wed Jun 21 01:29:53 2023 +0300
# Node ID bd2cc76ebe2367dc303e2746928b17ca8976b604
# Parent  b79ef48b91e45dba4bf850be6b2a2cc3b8834f5d
SSL: avoid using OpenSSL config in build directory (ticket #2404).

With this change, the NGX_OPENSSL_NO_CONFIG macro is defined when nginx
is asked to build OpenSSL itself.  And with this macro automatic loading
of OpenSSL configuration (from the build directory) is prevented unless
the OPENSSL_CONF environment variable is explicitly set.

Note that not loading configuration is broken in OpenSSL 1.1.1 and 1.1.1a
(fixed in OpenSSL 1.1.1b, see https://github.com/openssl/openssl/issues/7350).
If nginx is used to compile these OpenSSL versions, configuring nginx with
NGX_OPENSSL_NO_CONFIG explicitly set to 0 might be used as a workaround.

diff --git a/auto/lib/openssl/conf b/auto/lib/openssl/conf
--- a/auto/lib/openssl/conf
+++ b/auto/lib/openssl/conf
@@ -8,6 +8,8 @@ if [ $OPENSSL != NONE ]; then
     have=NGX_OPENSSL . auto/have
     have=NGX_SSL . auto/have
 
+    have=NGX_OPENSSL_NO_CONFIG . auto/have
+
     if [ $USE_OPENSSL_QUIC = YES ]; then
         have=NGX_QUIC . auto/have
         have=NGX_QUIC_OPENSSL_COMPAT . auto/have
diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -142,8 +142,19 @@ ngx_ssl_init(ngx_log_t *log)
 {
 #if (OPENSSL_INIT_LOAD_CONFIG && !defined LIBRESSL_VERSION_NUMBER)
 
+    uint64_t                opts;
     OPENSSL_INIT_SETTINGS  *init;
 
+    opts = OPENSSL_INIT_LOAD_CONFIG;
+
+#if (NGX_OPENSSL_NO_CONFIG)
+
+    if (getenv("OPENSSL_CONF") == NULL) {
+        opts = OPENSSL_INIT_NO_LOAD_CONFIG;
+    }
+
+#endif
+
     init = OPENSSL_INIT_new();
     if (init == NULL) {
         ngx_ssl_error(NGX_LOG_ALERT, log, 0, "OPENSSL_INIT_new() failed");
@@ -158,7 +169,7 @@ ngx_ssl_init(ngx_log_t *log)
     }
 #endif
 
-    if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, init) == 0) {
+    if (OPENSSL_init_ssl(opts, init) == 0) {
         ngx_ssl_error(NGX_LOG_ALERT, log, 0, "OPENSSL_init_ssl() failed");
         return NGX_ERROR;
     }
@@ -174,6 +185,14 @@ ngx_ssl_init(ngx_log_t *log)
 
 #else
 
+#if (NGX_OPENSSL_NO_CONFIG)
+
+    if (getenv("OPENSSL_CONF") == NULL) {
+        OPENSSL_no_config();
+    }
+
+#endif
+
     OPENSSL_config("nginx");
 
     SSL_library_init();

From pluknet at nginx.com  Tue Jul 25 13:42:51 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Tue, 25 Jul 2023 13:42:51 +0000
Subject: [nginx] Contrib: vim syntax,
 update core and 3rd party module directives.
Message-ID: <hg.904c99bede17.1690292571.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/904c99bede17
branches:  
changeset: 9135:904c99bede17
user:      Gena Makhomed <gmm at csdoc.com>
date:      Mon Jul 24 18:04:41 2023 +0300
description:
Contrib: vim syntax, update core and 3rd party module directives.

List of 3rd party modules github repositories was obtained from
https://github.com/freebsd/freebsd-ports/blob/main/www/nginx-devel/Makefile.extmod

diffstat:

 contrib/vim/syntax/nginx.vim |  217 ++++++++++++++++++------------------------
 1 files changed, 93 insertions(+), 124 deletions(-)

diffs (465 lines):

diff -r c209dc4eed17 -r 904c99bede17 contrib/vim/syntax/nginx.vim
--- a/contrib/vim/syntax/nginx.vim	Wed Jul 19 05:09:23 2023 +0300
+++ b/contrib/vim/syntax/nginx.vim	Mon Jul 24 18:04:41 2023 +0300
@@ -65,12 +65,12 @@ syn match ngxListenComment '#.*$'
     \ contained
     \ nextgroup=@ngxListenParams skipwhite skipempty
 syn keyword ngxListenOptions contained
-    \ default_server ssl http2 proxy_protocol
+    \ default_server ssl quic proxy_protocol
     \ setfib fastopen backlog rcvbuf sndbuf accept_filter deferred bind
     \ ipv6only reuseport so_keepalive
     \ nextgroup=@ngxListenParams skipwhite skipempty
 syn keyword ngxListenOptionsDeprecated contained
-    \ spdy
+    \ http2
     \ nextgroup=@ngxListenParams skipwhite skipempty
 syn cluster ngxListenParams
     \ contains=ngxListenParam,ngxListenString,ngxListenComment
@@ -90,7 +90,6 @@ syn keyword ngxDirectiveBlock contained 
 syn keyword ngxDirectiveBlock contained geo
 syn keyword ngxDirectiveBlock contained map
 syn keyword ngxDirectiveBlock contained split_clients
-syn keyword ngxDirectiveBlock contained match
 
 syn keyword ngxDirectiveImportant contained include
 syn keyword ngxDirectiveImportant contained root
@@ -113,7 +112,6 @@ syn keyword ngxDirectiveError contained 
 
 syn keyword ngxDirectiveDeprecated contained proxy_downstream_buffer
 syn keyword ngxDirectiveDeprecated contained proxy_upstream_buffer
-syn keyword ngxDirectiveDeprecated contained ssl
 syn keyword ngxDirectiveDeprecated contained http2_idle_timeout
 syn keyword ngxDirectiveDeprecated contained http2_max_field_size
 syn keyword ngxDirectiveDeprecated contained http2_max_header_size
@@ -136,7 +134,6 @@ syn keyword ngxDirective contained alias
 syn keyword ngxDirective contained allow
 syn keyword ngxDirective contained ancient_browser
 syn keyword ngxDirective contained ancient_browser_value
-syn keyword ngxDirective contained api
 syn keyword ngxDirective contained auth_basic
 syn keyword ngxDirective contained auth_basic_user_file
 syn keyword ngxDirective contained auth_delay
@@ -144,15 +141,6 @@ syn keyword ngxDirective contained auth_
 syn keyword ngxDirective contained auth_http_header
 syn keyword ngxDirective contained auth_http_pass_client_cert
 syn keyword ngxDirective contained auth_http_timeout
-syn keyword ngxDirective contained auth_jwt
-syn keyword ngxDirective contained auth_jwt_claim_set
-syn keyword ngxDirective contained auth_jwt_header_set
-syn keyword ngxDirective contained auth_jwt_key_cache
-syn keyword ngxDirective contained auth_jwt_key_file
-syn keyword ngxDirective contained auth_jwt_key_request
-syn keyword ngxDirective contained auth_jwt_leeway
-syn keyword ngxDirective contained auth_jwt_require
-syn keyword ngxDirective contained auth_jwt_type
 syn keyword ngxDirective contained auth_request
 syn keyword ngxDirective contained auth_request_set
 syn keyword ngxDirective contained autoindex
@@ -193,8 +181,6 @@ syn keyword ngxDirective contained error
 syn keyword ngxDirective contained etag
 syn keyword ngxDirective contained eventport_events
 syn keyword ngxDirective contained expires
-syn keyword ngxDirective contained f4f
-syn keyword ngxDirective contained f4f_buffer_size
 syn keyword ngxDirective contained fastcgi_bind
 syn keyword ngxDirective contained fastcgi_buffer_size
 syn keyword ngxDirective contained fastcgi_buffering
@@ -211,7 +197,6 @@ syn keyword ngxDirective contained fastc
 syn keyword ngxDirective contained fastcgi_cache_methods
 syn keyword ngxDirective contained fastcgi_cache_min_uses
 syn keyword ngxDirective contained fastcgi_cache_path
-syn keyword ngxDirective contained fastcgi_cache_purge
 syn keyword ngxDirective contained fastcgi_cache_revalidate
 syn keyword ngxDirective contained fastcgi_cache_use_stale
 syn keyword ngxDirective contained fastcgi_cache_valid
@@ -295,14 +280,7 @@ syn keyword ngxDirective contained gzip_
 syn keyword ngxDirective contained gzip_vary
 syn keyword ngxDirective contained gzip_window
 syn keyword ngxDirective contained hash
-syn keyword ngxDirective contained health_check
-syn keyword ngxDirective contained health_check_timeout
-syn keyword ngxDirective contained hls
-syn keyword ngxDirective contained hls_buffers
-syn keyword ngxDirective contained hls_forward_args
-syn keyword ngxDirective contained hls_fragment
-syn keyword ngxDirective contained hls_mp4_buffer_size
-syn keyword ngxDirective contained hls_mp4_max_buffer_size
+syn keyword ngxDirective contained http2
 syn keyword ngxDirective contained http2_body_preread_size
 syn keyword ngxDirective contained http2_chunk_size
 syn keyword ngxDirective contained http2_max_concurrent_pushes
@@ -312,6 +290,10 @@ syn keyword ngxDirective contained http2
 syn keyword ngxDirective contained http2_push_preload
 syn keyword ngxDirective contained http2_recv_buffer_size
 syn keyword ngxDirective contained http2_streams_index_size
+syn keyword ngxDirective contained http3
+syn keyword ngxDirective contained http3_hq
+syn keyword ngxDirective contained http3_max_concurrent_streams
+syn keyword ngxDirective contained http3_stream_buffer_size
 syn keyword ngxDirective contained if_modified_since
 syn keyword ngxDirective contained ignore_invalid_headers
 syn keyword ngxDirective contained image_filter
@@ -342,21 +324,20 @@ syn keyword ngxDirective contained js_fi
 syn keyword ngxDirective contained js_header_filter
 syn keyword ngxDirective contained js_import
 syn keyword ngxDirective contained js_path
+syn keyword ngxDirective contained js_preload_object
 syn keyword ngxDirective contained js_preread
 syn keyword ngxDirective contained js_set
+syn keyword ngxDirective contained js_shared_dict_zone
 syn keyword ngxDirective contained js_var
 syn keyword ngxDirective contained keepalive
 syn keyword ngxDirective contained keepalive_disable
 syn keyword ngxDirective contained keepalive_requests
 syn keyword ngxDirective contained keepalive_time
 syn keyword ngxDirective contained keepalive_timeout
-syn keyword ngxDirective contained keyval
-syn keyword ngxDirective contained keyval_zone
 syn keyword ngxDirective contained kqueue_changes
 syn keyword ngxDirective contained kqueue_events
 syn keyword ngxDirective contained large_client_header_buffers
 syn keyword ngxDirective contained least_conn
-syn keyword ngxDirective contained least_time
 syn keyword ngxDirective contained limit_conn
 syn keyword ngxDirective contained limit_conn_dry_run
 syn keyword ngxDirective contained limit_conn_log_level
@@ -400,14 +381,11 @@ syn keyword ngxDirective contained moder
 syn keyword ngxDirective contained modern_browser_value
 syn keyword ngxDirective contained mp4
 syn keyword ngxDirective contained mp4_buffer_size
-syn keyword ngxDirective contained mp4_limit_rate
-syn keyword ngxDirective contained mp4_limit_rate_after
 syn keyword ngxDirective contained mp4_max_buffer_size
 syn keyword ngxDirective contained mp4_start_key_frame
 syn keyword ngxDirective contained msie_padding
 syn keyword ngxDirective contained msie_refresh
 syn keyword ngxDirective contained multi_accept
-syn keyword ngxDirective contained ntlm
 syn keyword ngxDirective contained open_file_cache
 syn keyword ngxDirective contained open_file_cache_errors
 syn keyword ngxDirective contained open_file_cache_events
@@ -450,7 +428,6 @@ syn keyword ngxDirective contained proxy
 syn keyword ngxDirective contained proxy_cache_methods
 syn keyword ngxDirective contained proxy_cache_min_uses
 syn keyword ngxDirective contained proxy_cache_path
-syn keyword ngxDirective contained proxy_cache_purge
 syn keyword ngxDirective contained proxy_cache_revalidate
 syn keyword ngxDirective contained proxy_cache_use_stale
 syn keyword ngxDirective contained proxy_cache_valid
@@ -488,7 +465,6 @@ syn keyword ngxDirective contained proxy
 syn keyword ngxDirective contained proxy_responses
 syn keyword ngxDirective contained proxy_send_lowat
 syn keyword ngxDirective contained proxy_send_timeout
-syn keyword ngxDirective contained proxy_session_drop
 syn keyword ngxDirective contained proxy_set_body
 syn keyword ngxDirective contained proxy_set_header
 syn keyword ngxDirective contained proxy_smtp_auth
@@ -513,7 +489,11 @@ syn keyword ngxDirective contained proxy
 syn keyword ngxDirective contained proxy_temp_path
 syn keyword ngxDirective contained proxy_timeout
 syn keyword ngxDirective contained proxy_upload_rate
-syn keyword ngxDirective contained queue
+syn keyword ngxDirective contained quic_active_connection_id_limit
+syn keyword ngxDirective contained quic_bpf
+syn keyword ngxDirective contained quic_gso
+syn keyword ngxDirective contained quic_host_key
+syn keyword ngxDirective contained quic_retry
 syn keyword ngxDirective contained random
 syn keyword ngxDirective contained random_index
 syn keyword ngxDirective contained read_ahead
@@ -544,7 +524,6 @@ syn keyword ngxDirective contained scgi_
 syn keyword ngxDirective contained scgi_cache_methods
 syn keyword ngxDirective contained scgi_cache_min_uses
 syn keyword ngxDirective contained scgi_cache_path
-syn keyword ngxDirective contained scgi_cache_purge
 syn keyword ngxDirective contained scgi_cache_revalidate
 syn keyword ngxDirective contained scgi_cache_use_stale
 syn keyword ngxDirective contained scgi_cache_valid
@@ -583,9 +562,6 @@ syn keyword ngxDirective contained serve
 syn keyword ngxDirective contained server_names_hash_bucket_size
 syn keyword ngxDirective contained server_names_hash_max_size
 syn keyword ngxDirective contained server_tokens
-syn keyword ngxDirective contained session_log
-syn keyword ngxDirective contained session_log_format
-syn keyword ngxDirective contained session_log_zone
 syn keyword ngxDirective contained set_real_ip_from
 syn keyword ngxDirective contained slice
 syn keyword ngxDirective contained smtp_auth
@@ -633,11 +609,6 @@ syn keyword ngxDirective contained ssl_t
 syn keyword ngxDirective contained ssl_verify_client
 syn keyword ngxDirective contained ssl_verify_depth
 syn keyword ngxDirective contained starttls
-syn keyword ngxDirective contained state
-syn keyword ngxDirective contained status
-syn keyword ngxDirective contained status_format
-syn keyword ngxDirective contained status_zone
-syn keyword ngxDirective contained sticky
 syn keyword ngxDirective contained stub_status
 syn keyword ngxDirective contained sub_filter
 syn keyword ngxDirective contained sub_filter_last_modified
@@ -680,7 +651,6 @@ syn keyword ngxDirective contained uwsgi
 syn keyword ngxDirective contained uwsgi_cache_methods
 syn keyword ngxDirective contained uwsgi_cache_min_uses
 syn keyword ngxDirective contained uwsgi_cache_path
-syn keyword ngxDirective contained uwsgi_cache_purge
 syn keyword ngxDirective contained uwsgi_cache_revalidate
 syn keyword ngxDirective contained uwsgi_cache_use_stale
 syn keyword ngxDirective contained uwsgi_cache_valid
@@ -744,6 +714,62 @@ syn keyword ngxDirective contained xslt_
 syn keyword ngxDirective contained xslt_stylesheet
 syn keyword ngxDirective contained xslt_types
 syn keyword ngxDirective contained zone
+
+" nginx-plus commercial extensions directives
+
+syn keyword ngxDirectiveBlock contained match
+syn keyword ngxDirectiveBlock contained otel_exporter
+
+syn keyword ngxDirective contained api
+syn keyword ngxDirective contained auth_jwt
+syn keyword ngxDirective contained auth_jwt_claim_set
+syn keyword ngxDirective contained auth_jwt_header_set
+syn keyword ngxDirective contained auth_jwt_key_cache
+syn keyword ngxDirective contained auth_jwt_key_file
+syn keyword ngxDirective contained auth_jwt_key_request
+syn keyword ngxDirective contained auth_jwt_leeway
+syn keyword ngxDirective contained auth_jwt_require
+syn keyword ngxDirective contained auth_jwt_type
+syn keyword ngxDirective contained f4f
+syn keyword ngxDirective contained f4f_buffer_size
+syn keyword ngxDirective contained fastcgi_cache_purge
+syn keyword ngxDirective contained health_check
+syn keyword ngxDirective contained health_check_timeout
+syn keyword ngxDirective contained hls
+syn keyword ngxDirective contained hls_buffers
+syn keyword ngxDirective contained hls_forward_args
+syn keyword ngxDirective contained hls_fragment
+syn keyword ngxDirective contained hls_mp4_buffer_size
+syn keyword ngxDirective contained hls_mp4_max_buffer_size
+syn keyword ngxDirective contained internal_redirect
+syn keyword ngxDirective contained keyval
+syn keyword ngxDirective contained keyval_zone
+syn keyword ngxDirective contained least_time
+syn keyword ngxDirective contained mp4_limit_rate
+syn keyword ngxDirective contained mp4_limit_rate_after
+syn keyword ngxDirective contained mqtt
+syn keyword ngxDirective contained mqtt_preread
+syn keyword ngxDirective contained mqtt_rewrite_buffer_size
+syn keyword ngxDirective contained mqtt_set_connect
+syn keyword ngxDirective contained ntlm
+syn keyword ngxDirective contained otel_service_name
+syn keyword ngxDirective contained otel_span_attr
+syn keyword ngxDirective contained otel_span_name
+syn keyword ngxDirective contained otel_trace
+syn keyword ngxDirective contained otel_trace_context
+syn keyword ngxDirective contained proxy_cache_purge
+syn keyword ngxDirective contained proxy_session_drop
+syn keyword ngxDirective contained queue
+syn keyword ngxDirective contained scgi_cache_purge
+syn keyword ngxDirective contained session_log
+syn keyword ngxDirective contained session_log_format
+syn keyword ngxDirective contained session_log_zone
+syn keyword ngxDirective contained state
+syn keyword ngxDirective contained status
+syn keyword ngxDirective contained status_format
+syn keyword ngxDirective contained status_zone
+syn keyword ngxDirective contained sticky
+syn keyword ngxDirective contained uwsgi_cache_purge
 syn keyword ngxDirective contained zone_sync
 syn keyword ngxDirective contained zone_sync_buffers
 syn keyword ngxDirective contained zone_sync_connect_retry_interval
@@ -766,7 +792,6 @@ syn keyword ngxDirective contained zone_
 syn keyword ngxDirective contained zone_sync_ssl_verify_depth
 syn keyword ngxDirective contained zone_sync_timeout
 
-
 " 3rd party modules list taken from
 " https://github.com/freebsd/freebsd-ports/blob/main/www/nginx-devel/Makefile.extmod
 " ----------------------------------------------------------------------------------
@@ -837,52 +862,6 @@ syn keyword ngxDirectiveThirdParty conta
 " https://github.com/torden/ngx_cache_purge
 syn keyword ngxDirectiveThirdParty contained cache_purge_response_type
 
-" https://github.com/nginx-clojure/nginx-clojure
-syn keyword ngxDirectiveThirdParty contained access_handler_code
-syn keyword ngxDirectiveThirdParty contained access_handler_name
-syn keyword ngxDirectiveThirdParty contained access_handler_property
-syn keyword ngxDirectiveThirdParty contained access_handler_type
-syn keyword ngxDirectiveThirdParty contained always_read_body
-syn keyword ngxDirectiveThirdParty contained auto_upgrade_ws
-syn keyword ngxDirectiveThirdParty contained body_filter_code
-syn keyword ngxDirectiveThirdParty contained body_filter_name
-syn keyword ngxDirectiveThirdParty contained body_filter_property
-syn keyword ngxDirectiveThirdParty contained body_filter_type
-syn keyword ngxDirectiveThirdParty contained content_handler_code
-syn keyword ngxDirectiveThirdParty contained content_handler_name
-syn keyword ngxDirectiveThirdParty contained content_handler_property
-syn keyword ngxDirectiveThirdParty contained content_handler_type
-syn keyword ngxDirectiveThirdParty contained handler_code
-syn keyword ngxDirectiveThirdParty contained handler_name
-syn keyword ngxDirectiveThirdParty contained handler_type
-syn keyword ngxDirectiveThirdParty contained handlers_lazy_init
-syn keyword ngxDirectiveThirdParty contained header_filter_code
-syn keyword ngxDirectiveThirdParty contained header_filter_name
-syn keyword ngxDirectiveThirdParty contained header_filter_property
-syn keyword ngxDirectiveThirdParty contained header_filter_type
-syn keyword ngxDirectiveThirdParty contained jvm_classpath
-syn keyword ngxDirectiveThirdParty contained jvm_classpath_check
-syn keyword ngxDirectiveThirdParty contained jvm_exit_handler_code
-syn keyword ngxDirectiveThirdParty contained jvm_exit_handler_name
-syn keyword ngxDirectiveThirdParty contained jvm_handler_type
-syn keyword ngxDirectiveThirdParty contained jvm_init_handler_code
-syn keyword ngxDirectiveThirdParty contained jvm_init_handler_name
-syn keyword ngxDirectiveThirdParty contained jvm_options
-syn keyword ngxDirectiveThirdParty contained jvm_path
-syn keyword ngxDirectiveThirdParty contained jvm_var
-syn keyword ngxDirectiveThirdParty contained jvm_workers
-syn keyword ngxDirectiveThirdParty contained log_handler_code
-syn keyword ngxDirectiveThirdParty contained log_handler_name
-syn keyword ngxDirectiveThirdParty contained log_handler_property
-syn keyword ngxDirectiveThirdParty contained log_handler_type
-syn keyword ngxDirectiveThirdParty contained max_balanced_tcp_connections
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_code
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_name
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_property
-syn keyword ngxDirectiveThirdParty contained rewrite_handler_type
-syn keyword ngxDirectiveThirdParty contained shared_map
-syn keyword ngxDirectiveThirdParty contained write_page_size
-
 " https://github.com/AirisX/nginx_cookie_flag_module
 syn keyword ngxDirectiveThirdParty contained set_cookie_flag
 
@@ -932,29 +911,6 @@ syn keyword ngxDirectiveThirdParty conta
 syn keyword ngxDirectiveThirdParty contained dynamic_state_file
 syn keyword ngxDirectiveThirdParty contained dynamic_upstream
 
-" https://github.com/ZigzagAK/ngx_dynamic_healthcheck
-syn keyword ngxDirectiveThirdParty contained check
-syn keyword ngxDirectiveThirdParty contained check_disable_host
-syn keyword ngxDirectiveThirdParty contained check_exclude_host
-syn keyword ngxDirectiveThirdParty contained check_persistent
-syn keyword ngxDirectiveThirdParty contained check_request_body
-syn keyword ngxDirectiveThirdParty contained check_request_headers
-syn keyword ngxDirectiveThirdParty contained check_request_uri
-syn keyword ngxDirectiveThirdParty contained check_response_body
-syn keyword ngxDirectiveThirdParty contained check_response_codes
-syn keyword ngxDirectiveThirdParty contained healthcheck
-syn keyword ngxDirectiveThirdParty contained healthcheck_buffer_size
-syn keyword ngxDirectiveThirdParty contained healthcheck_disable_host
-syn keyword ngxDirectiveThirdParty contained healthcheck_get
-syn keyword ngxDirectiveThirdParty contained healthcheck_persistent
-syn keyword ngxDirectiveThirdParty contained healthcheck_request_body
-syn keyword ngxDirectiveThirdParty contained healthcheck_request_headers
-syn keyword ngxDirectiveThirdParty contained healthcheck_request_uri
-syn keyword ngxDirectiveThirdParty contained healthcheck_response_body
-syn keyword ngxDirectiveThirdParty contained healthcheck_response_codes
-syn keyword ngxDirectiveThirdParty contained healthcheck_status
-syn keyword ngxDirectiveThirdParty contained healthcheck_update
-
 " https://github.com/openresty/encrypted-session-nginx-module
 syn keyword ngxDirectiveThirdParty contained encrypted_session_expires
 syn keyword ngxDirectiveThirdParty contained encrypted_session_iv
@@ -1004,6 +960,7 @@ syn keyword ngxDirectiveThirdParty conta
 syn keyword ngxDirectiveThirdParty contained auth_gss_realm
 syn keyword ngxDirectiveThirdParty contained auth_gss_service_ccache
 syn keyword ngxDirectiveThirdParty contained auth_gss_service_name
+syn keyword ngxDirectiveThirdParty contained auth_gss_zone_name
 
 " https://github.com/kvspb/nginx-auth-ldap
 syn keyword ngxDirectiveThirdParty contained auth_ldap
@@ -1033,6 +990,7 @@ syn keyword ngxDirectiveThirdParty conta
 
 " https://github.com/aperezdc/ngx-fancyindex
 syn keyword ngxDirectiveThirdParty contained fancyindex
+syn keyword ngxDirectiveThirdParty contained fancyindex_case_sensitive
 syn keyword ngxDirectiveThirdParty contained fancyindex_css_href
 syn keyword ngxDirectiveThirdParty contained fancyindex_default_sort
 syn keyword ngxDirectiveThirdParty contained fancyindex_directories_first
@@ -1121,6 +1079,7 @@ syn keyword ngxDirectiveThirdParty conta
 syn keyword ngxDirectiveThirdParty contained nchan_pubsub
 syn keyword ngxDirectiveThirdParty contained nchan_pubsub_channel_id
 syn keyword ngxDirectiveThirdParty contained nchan_pubsub_location
+syn keyword ngxDirectiveThirdParty contained nchan_redis_accurate_subscriber_count
 syn keyword ngxDirectiveThirdParty contained nchan_redis_cluster_check_interval
 syn keyword ngxDirectiveThirdParty contained nchan_redis_cluster_check_interval_backoff
 syn keyword ngxDirectiveThirdParty contained nchan_redis_cluster_check_interval_jitter
@@ -1138,6 +1097,11 @@ syn keyword ngxDirectiveThirdParty conta
 syn keyword ngxDirectiveThirdParty contained nchan_redis_discovered_ip_range_blacklist
 syn keyword ngxDirectiveThirdParty contained nchan_redis_fakesub_timer_interval
 syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_cache_timeout
+syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_keepalive_backoff
+syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_keepalive_jitter
+syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_keepalive_max
+syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_keepalive_min
+syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_keepalive_safety_margin
 syn keyword ngxDirectiveThirdParty contained nchan_redis_load_scripts_unconditionally
 syn keyword ngxDirectiveThirdParty contained nchan_redis_namespace
 syn keyword ngxDirectiveThirdParty contained nchan_redis_node_connect_timeout
@@ -1173,6 +1137,9 @@ syn keyword ngxDirectiveThirdParty conta
 syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_trusted_certificate
 syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_trusted_certificate_path
 syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_verify_certificate
+syn keyword ngxDirectiveThirdParty contained nchan_redis_upstream_stats
+syn keyword ngxDirectiveThirdParty contained nchan_redis_upstream_stats_disconnected_timeout
+syn keyword ngxDirectiveThirdParty contained nchan_redis_upstream_stats_enabled
 syn keyword ngxDirectiveThirdParty contained nchan_redis_url
 syn keyword ngxDirectiveThirdParty contained nchan_redis_username
 syn keyword ngxDirectiveThirdParty contained nchan_redis_wait_after_connecting
@@ -1323,6 +1290,7 @@ syn keyword ngxDirectiveThirdParty conta
 syn keyword ngxDirectiveThirdParty contained upload_progress_template
 
 " https://github.com/yaoweibin/nginx_upstream_check_module
+syn keyword ngxDirectiveThirdParty contained check
 syn keyword ngxDirectiveThirdParty contained check_fastcgi_param
 syn keyword ngxDirectiveThirdParty contained check_http_expect_alive
 syn keyword ngxDirectiveThirdParty contained check_http_send
@@ -1335,6 +1303,7 @@ syn keyword ngxDirectiveThirdParty conta
 syn keyword ngxDirectiveThirdParty contained upstream_fair_shm_size
 
 " https://github.com/ayty-adrianomartins/nginx-sticky-module-ng
+syn keyword ngxDirectiveThirdParty contained sticky_hide_cookie
 syn keyword ngxDirectiveThirdParty contained sticky_no_fallback
 
 " https://github.com/Novetta/nginx-video-thumbextractor-module
@@ -1421,6 +1390,8 @@ syn keyword ngxDirectiveThirdParty conta
 syn keyword ngxDirectiveThirdParty contained lua_socket_read_timeout
 syn keyword ngxDirectiveThirdParty contained lua_socket_send_lowat
 syn keyword ngxDirectiveThirdParty contained lua_socket_send_timeout
+syn keyword ngxDirectiveThirdParty contained lua_ssl_certificate
+syn keyword ngxDirectiveThirdParty contained lua_ssl_certificate_key
 syn keyword ngxDirectiveThirdParty contained lua_ssl_ciphers
 syn keyword ngxDirectiveThirdParty contained lua_ssl_conf_command
 syn keyword ngxDirectiveThirdParty contained lua_ssl_crl
@@ -1834,16 +1805,6 @@ syn keyword ngxDirectiveThirdParty conta
 syn keyword ngxDirectiveThirdParty contained slowfs_cache_valid
 syn keyword ngxDirectiveThirdParty contained slowfs_temp_path
 
-" https://github.com/kawakibi/ngx_small_light
-syn keyword ngxDirectiveThirdParty contained small_light
-syn keyword ngxDirectiveThirdParty contained small_light_buffer
-syn keyword ngxDirectiveThirdParty contained small_light_getparam_mode
-syn keyword ngxDirectiveThirdParty contained small_light_imlib2_temp_dir
-syn keyword ngxDirectiveThirdParty contained small_light_material_dir
-syn keyword ngxDirectiveThirdParty contained small_light_pattern_define
-syn keyword ngxDirectiveThirdParty contained small_light_radius_max
-syn keyword ngxDirectiveThirdParty contained small_light_sigma_max
-
 " https://github.com/openresty/srcache-nginx-module
 syn keyword ngxDirectiveThirdParty contained srcache_buffer
 syn keyword ngxDirectiveThirdParty contained srcache_default_expire
@@ -1980,6 +1941,14 @@ syn keyword ngxDirectiveThirdParty conta
 syn keyword ngxDirectiveThirdParty contained websockify_read_timeout
 syn keyword ngxDirectiveThirdParty contained websockify_send_timeout
 
+" https://github.com/vozlt/nginx-module-sts
+syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status
+syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status_average_method
+syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status_display
+syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status_display_format
+syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status_display_jsonp
+syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status_zone
+
 " highlight
 
 hi def link ngxComment Comment

From cnewton at netflix.com  Tue Jul 25 14:56:50 2023
From: cnewton at netflix.com (Chris Newton)
Date: Tue, 25 Jul 2023 07:56:50 -0700
Subject: extension to server_name directive - include top-level directory in
 search
Message-ID: <CA+q-vaYr+4oR4qW=tGEWT4QbZE75k8s-bNmhNpMLBf1SPTj=-w@mail.gmail.com>

The server_name directive allows for selecting which "server" block should
be used to process a request; a "server" being a very convenient way to
group a set of directives and location blocks into a separate and distinct
configuration entity. Currently, server_name selects the server based on
the value of the Host: header alone, which can be overly constraining.

The attached patch extends server_name to allow the first directory element
to be used as part of the selection process. When there is at least one
server_name with a '/' character, at ngx_http_find_virtual_server() time an
initial lookup into the virtual_names hash is made using a combination of
hostname/topleveldirectory - if no match is found, the hostname only lookup
is performed (as before), thereby allowing for specific directories to be
broken out into their own server blocks

>From a01fd492fb8935e9c9e0f4c703b9430c83c12590 Mon Sep 17 00:00:00 2001

From: Chris Newton <cnewton at netflix.com>

Date: Mon, 24 Jul 2023 19:32:03 +0000

Subject: [PATCH] virtual aliases


---

 src/http/ngx_http.c             |  2 ++

 src/http/ngx_http.h             |  1 +

 src/http/ngx_http_core_module.c |  4 +--

 src/http/ngx_http_request.c     | 54 ++++++++++++++++++++++++++++-----

 4 files changed, 51 insertions(+), 10 deletions(-)


diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c

index d835f896e..54ad9cb9e 100644

--- a/src/http/ngx_http.c

+++ b/src/http/ngx_http.c

@@ -70,6 +70,7 @@ static ngx_int_t ngx_http_add_addrs6(ngx_conf_t *cf,
ngx_http_port_t *hport,



 ngx_uint_t   ngx_http_max_module;



+ngx_uint_t  ngx_http_uses_complete_alias;



 ngx_http_output_header_filter_pt  ngx_http_top_header_filter;

 ngx_http_output_body_filter_pt    ngx_http_top_body_filter;

@@ -148,6 +149,7 @@ ngx_http_block(ngx_conf_t *cf, ngx_command_t *cmd, void
*conf)



     ngx_http_max_module = ngx_count_modules(cf->cycle, NGX_HTTP_MODULE);



+    ngx_http_uses_complete_alias = 0;



     /* the http main_conf context, it is the same in the all http contexts
*/



diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h

index e06464ebd..935cecc0f 100644

--- a/src/http/ngx_http.h

+++ b/src/http/ngx_http.h

@@ -189,6 +189,7 @@ extern ngx_module_t  ngx_http_module;



 extern ngx_str_t  ngx_http_html_default_types[];



+extern ngx_uint_t  ngx_http_uses_complete_alias;



 extern ngx_http_output_header_filter_pt  ngx_http_top_header_filter;

 extern ngx_http_output_body_filter_pt    ngx_http_top_body_filter;

diff --git a/src/http/ngx_http_core_module.c
b/src/http/ngx_http_core_module.c

index 97a91aee2..3179d65a1 100644

--- a/src/http/ngx_http_core_module.c

+++ b/src/http/ngx_http_core_module.c

@@ -4380,9 +4380,7 @@ ngx_http_core_server_name(ngx_conf_t *cf,
ngx_command_t *cmd, void *conf)

         }



         if (ngx_strchr(value[i].data, '/')) {

-            ngx_conf_log_error(NGX_LOG_WARN, cf, 0,

-                               "server name \"%V\" has suspicious symbols",

-                               &value[i]);

+            ngx_http_uses_complete_alias = 1;

         }



         sn = ngx_array_push(&cscf->server_names);

diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c

index bd2be5eac..189500885 100644

--- a/src/http/ngx_http_request.c

+++ b/src/http/ngx_http_request.c

@@ -2227,6 +2227,7 @@ ngx_http_set_virtual_server(ngx_http_request_t *r,
ngx_str_t *host)

     ngx_http_connection_t     *hc;

     ngx_http_core_loc_conf_t  *clcf;

     ngx_http_core_srv_conf_t  *cscf;

+    ngx_str_t                 *find_host = host;



 #if (NGX_SUPPRESS_WARN)

     cscf = NULL;

@@ -2250,7 +2251,11 @@ ngx_http_set_virtual_server(ngx_http_request_t *r,
ngx_str_t *host)

                 return NGX_ERROR;

             }

 #endif

-            return NGX_OK;

+            if (ngx_http_uses_complete_alias) {

+                find_host = hc->ssl_servername;

+            } else {

+                return NGX_OK;

+            }

         }

     }



@@ -2258,7 +2263,7 @@ ngx_http_set_virtual_server(ngx_http_request_t *r,
ngx_str_t *host)



     rc = ngx_http_find_virtual_server(r->connection,

                                       hc->addr_conf->virtual_names,

-                                      host, r, &cscf);

+                                      find_host, r, &cscf);



     if (rc == NGX_ERROR) {

         ngx_http_close_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);

@@ -2275,14 +2280,19 @@ ngx_http_set_virtual_server(ngx_http_request_t *r,
ngx_str_t *host)

             rc = NGX_OK;

         }



-        sscf = ngx_http_get_module_srv_conf(cscf->ctx,
ngx_http_ssl_module);

+        if (!ngx_http_uses_complete_alias

+            || hc->ssl_servername->len != host->len

+            || ngx_strncmp(hc->ssl_servername->data,

+                           host->data, host->len) != 0) {

+            sscf = ngx_http_get_module_srv_conf(cscf->ctx,
ngx_http_ssl_module);



-        if (sscf->verify) {

-            ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,

+            if (sscf->verify) {

+                ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,

                           "client attempted to request the server name "

                           "different from the one that was negotiated");

-            ngx_http_finalize_request(r, NGX_HTTP_MISDIRECTED_REQUEST);

-            return NGX_ERROR;

+                ngx_http_finalize_request(r, NGX_HTTP_MISDIRECTED_REQUEST);

+                return NGX_ERROR;

+            }

         }

     }



@@ -2314,6 +2324,36 @@ ngx_http_find_virtual_server(ngx_connection_t *c,

         return NGX_DECLINED;

     }



+    if (r && r->uri.len > 1 && r->pool) {

+        ngx_str_t   combined;

+        u_char     *p;

+

+        p = (u_char *)ngx_strchr(&r->uri.data[1], '/');

+        if (p) {

+            ngx_uint_t alias_len = p - r->uri.data;

+            if (alias_len > r->uri.len) {

+                /* No '/' in the path, so use entire thing */

+                alias_len = r->uri.len;

+            }

+            combined.len = host->len + alias_len;

+

+            combined.data = ngx_pnalloc(r->pool, combined.len);

+            if (combined.data == NULL) {

+                return NGX_ERROR;

+            }

+            ngx_memcpy(&combined.data[0], host->data, host->len);

+            ngx_memcpy(&combined.data[host->len], r->uri.data, alias_len);

+

+            cscf = ngx_hash_find_combined(&virtual_names->names,

+                                  ngx_hash_key(combined.data,
combined.len),

+                                  combined.data, combined.len);

+            if (cscf) {

+                *cscfp = cscf;

+                return NGX_OK;

+            }

+        }

+    }

+

     cscf = ngx_hash_find_combined(&virtual_names->names,

                                   ngx_hash_key(host->data, host->len),

                                   host->data, host->len);

-- 

2.40.1




*Chris Newton* (he/him/his)
Open Connect | Content Delivery Architecture M: 805.444.0573 |
cnewton at netflix.com
111 Albright Way | Los Gatos, CA 95032
<https://maps.google.com/?q=111+Albright+Way%C2%A0%7C++Los+Gatos,+CA+95032&entry=gmail&source=g>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20230725/c1f74120/attachment-0001.htm>

From mdounin at mdounin.ru  Tue Jul 25 20:00:53 2023
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 25 Jul 2023 23:00:53 +0300
Subject: extension to server_name directive - include top-level directory
 in search
In-Reply-To: <CA+q-vaYr+4oR4qW=tGEWT4QbZE75k8s-bNmhNpMLBf1SPTj=-w@mail.gmail.com>
References: <CA+q-vaYr+4oR4qW=tGEWT4QbZE75k8s-bNmhNpMLBf1SPTj=-w@mail.gmail.com>
Message-ID: <ZMAp9UOL5umwyXWp@mdounin.ru>

Hello!

On Tue, Jul 25, 2023 at 07:56:50AM -0700, Chris Newton via nginx-devel wrote:

> The server_name directive allows for selecting which "server" block should
> be used to process a request; a "server" being a very convenient way to
> group a set of directives and location blocks into a separate and distinct
> configuration entity. Currently, server_name selects the server based on
> the value of the Host: header alone, which can be overly constraining.
> 
> The attached patch extends server_name to allow the first directory element
> to be used as part of the selection process. When there is at least one
> server_name with a '/' character, at ngx_http_find_virtual_server() time an
> initial lookup into the virtual_names hash is made using a combination of
> hostname/topleveldirectory - if no match is found, the hostname only lookup
> is performed (as before), thereby allowing for specific directories to be
> broken out into their own server blocks

This somewhat contradicts the configuration model as currently 
used in nginx: for different name-based virtual hosts, "server" 
blocks are used as a basic configuration entity, and for different 
URI prefixes within these virtual hosts there are "location" 
blocks.

With the change suggested, these are combined into multiple 
"location-based-server" blocks, and I see at least the following 
issues with this approach:

- It would be really non-trivial to find out how a particular 
  request is handled when looking into a configuration.  In 
  particular, even with the Host name explicitly known, full nginx 
  configuration would be needed to find out correct "server" 
  block.

- There will be at least three different server blocks involved 
  during a request handling: the default one for a listening 
  socket (for early connection-specific options), the default one 
  for a host (for connection-specific options after SNI name is 
  know), and the one matched for a request URI.

Also, I see at least the following issues in the implementation:

- The code uses the ngx_http_uses_complete_alias global variable, 
  which is set during configuration parsing.  This is not going to 
  work in general, for example, the global variable will be 
  incorrect on a worker process respawn after configuration parsing 
  errors.

- Handling assumes that r->uri for a request is available during 
  the ngx_http_find_virtual_server() call, which might not be the 
  case, for example, for HTTP/2, since order of pseudo-headers is 
  not guaranteed.

Overall, a better idea might be to focus on what's the problem you 
are trying to solve, and how to solve it within nginx 
configuration model.

[...]

-- 
Maxim Dounin
http://mdounin.ru/

From arut at nginx.com  Thu Jul 27 12:42:35 2023
From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=)
Date: Thu, 27 Jul 2023 16:42:35 +0400
Subject: [PATCH 0 of 4] QUIC performance optimizations
Message-ID: <patchbomb.1690461755@plentiful-waterbuck>

Hello,

- patches 1 & 2: ACK sending optimizations
- patch 3: GSO congestion fix
- patch 4: sendmmsg() instead of sendmsg() for GSO

--
Roman Arutyunyan

From arut at nginx.com  Thu Jul 27 12:42:36 2023
From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=)
Date: Thu, 27 Jul 2023 16:42:36 +0400
Subject: [PATCH 1 of 4] QUIC: optimized ACK delay
In-Reply-To: <patchbomb.1690461755@plentiful-waterbuck>
References: <patchbomb.1690461755@plentiful-waterbuck>
Message-ID: <2fd16fc76920ef0b8ea2.1690461756@plentiful-waterbuck>

# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1690461437 -14400
#      Thu Jul 27 16:37:17 2023 +0400
# Node ID 2fd16fc76920ef0b8ea2fa64858934e38c4477c5
# Parent  904c99bede1770d92566b56939c5b6ec85f05b55
QUIC: optimized ACK delay.

Previously ACK was not generated if max_ack_delay was not yet expired and the
number of unacknowledged ack-eliciting packets was less than two, as allowed by
RFC 9000 13.2.1-13.2.2.  However this only makes sense to avoid sending ACK-only
packets, as explained by the RFC:

  On the other hand, reducing the frequency of packets that carry only
  acknowledgments reduces packet transmission and processing cost at both
  endpoints.

Now ACK is delayed only if output frame queue is empty.  Otherwise ACK is sent
immediately, which significantly improves QUIC performance with certain tests.

diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c
--- a/src/event/quic/ngx_event_quic_ack.c
+++ b/src/event/quic/ngx_event_quic_ack.c
@@ -1171,7 +1171,8 @@ ngx_quic_generate_ack(ngx_connection_t *
         delay = ngx_current_msec - ctx->ack_delay_start;
         qc = ngx_quic_get_connection(c);
 
-        if (ctx->send_ack < NGX_QUIC_MAX_ACK_GAP
+        if (ngx_queue_empty(&ctx->frames)
+            && ctx->send_ack < NGX_QUIC_MAX_ACK_GAP
             && delay < qc->tp.max_ack_delay)
         {
             if (!qc->push.timer_set && !qc->closing) {

From arut at nginx.com  Thu Jul 27 12:42:37 2023
From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=)
Date: Thu, 27 Jul 2023 16:42:37 +0400
Subject: [PATCH 2 of 4] QUIC: always add ACK frame to the queue head
In-Reply-To: <patchbomb.1690461755@plentiful-waterbuck>
References: <patchbomb.1690461755@plentiful-waterbuck>
Message-ID: <0d12ada84c168c62e9ba.1690461757@plentiful-waterbuck>

# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1690461509 -14400
#      Thu Jul 27 16:38:29 2023 +0400
# Node ID 0d12ada84c168c62e9bae847af2725641da583d0
# Parent  2fd16fc76920ef0b8ea2fa64858934e38c4477c5
QUIC: always add ACK frame to the queue head.

Previously it was added to the tail as all other frames.  However, if the
amount of queued data is large, it could delay the delivery of ACK, which
could trigger frames retransmissions and slow down the connection.

diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -1175,7 +1175,9 @@ ngx_quic_send_ack(ngx_connection_t *c, n
     frame->u.ack.range_count = ctx->nranges;
     frame->u.ack.first_range = ctx->first_range;
 
-    ngx_quic_queue_frame(qc, frame);
+    ngx_queue_insert_head(&ctx->frames, &frame->queue);
+
+    frame->len = ngx_quic_create_frame(NULL, frame);
 
     return NGX_OK;
 }

From arut at nginx.com  Thu Jul 27 12:42:38 2023
From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=)
Date: Thu, 27 Jul 2023 16:42:38 +0400
Subject: [PATCH 3 of 4] QUIC: fixed congesion control in GSO mode
In-Reply-To: <patchbomb.1690461755@plentiful-waterbuck>
References: <patchbomb.1690461755@plentiful-waterbuck>
Message-ID: <d8a6ec55938e9a4f1a84.1690461758@plentiful-waterbuck>

# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1690450542 -14400
#      Thu Jul 27 13:35:42 2023 +0400
# Node ID d8a6ec55938e9a4f1a84c825d9c6abe13aa8b791
# Parent  0d12ada84c168c62e9bae847af2725641da583d0
QUIC: fixed congesion control in GSO mode.

In non-GSO mode, a datagram is sent if congestion window is not exceeded by the
time of send.  The window could be exceeded by a small amount after the send.
In GSO mode, congestion window was checked in a similar way, but for all
concatenated datagrams as a whole.  This could result in exceeding congestion
window by a lot.  Now congestion window is checked for every datagram in GSO
mode as well.

diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -358,7 +358,7 @@ ngx_quic_create_segments(ngx_connection_
 
         len = ngx_min(segsize, (size_t) (end - p));
 
-        if (len && cg->in_flight < cg->window) {
+        if (len && cg->in_flight + (p - dst) < cg->window) {
 
             n = ngx_quic_output_packet(c, ctx, p, len, len);
             if (n == NGX_ERROR) {

From arut at nginx.com  Thu Jul 27 12:42:39 2023
From: arut at nginx.com (=?iso-8859-1?q?Roman_Arutyunyan?=)
Date: Thu, 27 Jul 2023 16:42:39 +0400
Subject: [PATCH 4 of 4] QUIC: use sendmmsg() with GSO
In-Reply-To: <patchbomb.1690461755@plentiful-waterbuck>
References: <patchbomb.1690461755@plentiful-waterbuck>
Message-ID: <4f078be6e2ed08643371.1690461759@plentiful-waterbuck>

# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1690460906 -14400
#      Thu Jul 27 16:28:26 2023 +0400
# Node ID 4f078be6e2ed08643371a3956f5f18f2357a38db
# Parent  d8a6ec55938e9a4f1a84c825d9c6abe13aa8b791
QUIC: use sendmmsg() with GSO.

This syscall appeared before GSO and allows to send more datagrams with a
single call.

diff --git a/auto/os/linux b/auto/os/linux
--- a/auto/os/linux
+++ b/auto/os/linux
@@ -291,4 +291,16 @@ ngx_feature_test="socklen_t optlen = siz
 . auto/feature
 
 
+ngx_feature="sendmmsg()"
+ngx_feature_name="NGX_HAVE_SENDMMSG"
+ngx_feature_run=no
+ngx_feature_incs="#include <sys/socket.h>
+                  #include <sys/uio.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="struct mmsghdr msg[UIO_MAXIOV];
+                  sendmmsg(0, msg, UIO_MAXIOV, 0);"
+. auto/feature
+
+
 CC_AUX_FLAGS="$cc_aux_flags -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64"
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -42,11 +42,9 @@ static ngx_int_t ngx_quic_create_datagra
 static void ngx_quic_commit_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx);
 static void ngx_quic_revert_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx,
     uint64_t pnum);
-#if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL))
+#if (NGX_HAVE_UDP_SEGMENT && NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_HAVE_SENDMMSG)
 static ngx_uint_t ngx_quic_allow_segmentation(ngx_connection_t *c);
 static ngx_int_t ngx_quic_create_segments(ngx_connection_t *c);
-static ssize_t ngx_quic_send_segments(ngx_connection_t *c, u_char *buf,
-    size_t len, struct sockaddr *sockaddr, socklen_t socklen, size_t segment);
 #endif
 static ssize_t ngx_quic_output_packet(ngx_connection_t *c,
     ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min);
@@ -91,7 +89,7 @@ ngx_quic_output(ngx_connection_t *c)
 
     in_flight = cg->in_flight;
 
-#if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL))
+#if (NGX_HAVE_UDP_SEGMENT && NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_HAVE_SENDMMSG)
     if (ngx_quic_allow_segmentation(c)) {
         rc = ngx_quic_create_segments(c);
     } else
@@ -264,7 +262,7 @@ ngx_quic_revert_send(ngx_connection_t *c
 }
 
 
-#if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL))
+#if (NGX_HAVE_UDP_SEGMENT && NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_HAVE_SENDMMSG)
 
 static ngx_uint_t
 ngx_quic_allow_segmentation(ngx_connection_t *c)
@@ -324,16 +322,28 @@ ngx_quic_allow_segmentation(ngx_connecti
 static ngx_int_t
 ngx_quic_create_segments(ngx_connection_t *c)
 {
-    size_t                  len, segsize;
+    size_t                  len, segsize, clen;
     ssize_t                 n;
-    u_char                 *p, *end;
+    u_char                 *p, *start, *end;
+    uint16_t               *valp;
     uint64_t                preserved_pnum;
-    ngx_uint_t              nseg;
+    ngx_err_t               err;
+    ngx_uint_t              nseg, nmsg;
+    struct msghdr           msg;
+    struct cmsghdr         *cmsg;
     ngx_quic_path_t        *path;
     ngx_quic_send_ctx_t    *ctx;
     ngx_quic_congestion_t  *cg;
     ngx_quic_connection_t  *qc;
-    static u_char           dst[NGX_QUIC_MAX_UDP_SEGMENT_BUF];
+#if (NGX_HAVE_ADDRINFO_CMSG)
+    char                    msg_control[CMSG_SPACE(sizeof(uint16_t))
+                                        + CMSG_SPACE(sizeof(ngx_addrinfo_t))];
+#else
+    char                    msg_control[CMSG_SPACE(sizeof(uint16_t))];
+#endif
+    struct iovec            iovs[UIO_MAXIOV];
+    static u_char           dst[UIO_MAXIOV * NGX_QUIC_MAX_UDP_SEGMENT_BUF];
+    struct mmsghdr          msgs[UIO_MAXIOV];
 
     qc = ngx_quic_get_connection(c);
     cg = &qc->congestion;
@@ -347,94 +357,12 @@ ngx_quic_create_segments(ngx_connection_
 
     segsize = ngx_min(qc->ctp.max_udp_payload_size,
                       NGX_QUIC_MAX_UDP_SEGMENT_BUF);
-    p = dst;
-    end = dst + sizeof(dst);
-
-    nseg = 0;
-
-    preserved_pnum = ctx->pnum;
-
-    for ( ;; ) {
-
-        len = ngx_min(segsize, (size_t) (end - p));
-
-        if (len && cg->in_flight + (p - dst) < cg->window) {
-
-            n = ngx_quic_output_packet(c, ctx, p, len, len);
-            if (n == NGX_ERROR) {
-                return NGX_ERROR;
-            }
-
-            if (n) {
-                p += n;
-                nseg++;
-            }
-
-        } else {
-            n = 0;
-        }
-
-        if (p == dst) {
-            break;
-        }
-
-        if (n == 0 || nseg == NGX_QUIC_MAX_SEGMENTS) {
-            n = ngx_quic_send_segments(c, dst, p - dst, path->sockaddr,
-                                       path->socklen, segsize);
-            if (n == NGX_ERROR) {
-                return NGX_ERROR;
-            }
-
-            if (n == NGX_AGAIN) {
-                ngx_quic_revert_send(c, ctx, preserved_pnum);
-
-                ngx_add_timer(&qc->push, NGX_QUIC_SOCKET_RETRY_DELAY);
-                break;
-            }
-
-            ngx_quic_commit_send(c, ctx);
-
-            path->sent += n;
-
-            p = dst;
-            nseg = 0;
-            preserved_pnum = ctx->pnum;
-        }
-    }
-
-    return NGX_OK;
-}
-
-
-static ssize_t
-ngx_quic_send_segments(ngx_connection_t *c, u_char *buf, size_t len,
-    struct sockaddr *sockaddr, socklen_t socklen, size_t segment)
-{
-    size_t           clen;
-    ssize_t          n;
-    uint16_t        *valp;
-    struct iovec     iov;
-    struct msghdr    msg;
-    struct cmsghdr  *cmsg;
-
-#if (NGX_HAVE_ADDRINFO_CMSG)
-    char             msg_control[CMSG_SPACE(sizeof(uint16_t))
-                             + CMSG_SPACE(sizeof(ngx_addrinfo_t))];
-#else
-    char             msg_control[CMSG_SPACE(sizeof(uint16_t))];
-#endif
 
     ngx_memzero(&msg, sizeof(struct msghdr));
     ngx_memzero(msg_control, sizeof(msg_control));
 
-    iov.iov_len = len;
-    iov.iov_base = buf;
-
-    msg.msg_iov = &iov;
-    msg.msg_iovlen = 1;
-
-    msg.msg_name = sockaddr;
-    msg.msg_namelen = socklen;
+    msg.msg_name = path->sockaddr;
+    msg.msg_namelen = path->socklen;
 
     msg.msg_control = msg_control;
     msg.msg_controllen = sizeof(msg_control);
@@ -448,7 +376,7 @@ ngx_quic_send_segments(ngx_connection_t 
     clen = CMSG_SPACE(sizeof(uint16_t));
 
     valp = (void *) CMSG_DATA(cmsg);
-    *valp = segment;
+    *valp = segsize;
 
 #if (NGX_HAVE_ADDRINFO_CMSG)
     if (c->listening && c->listening->wildcard && c->local_sockaddr) {
@@ -459,14 +387,100 @@ ngx_quic_send_segments(ngx_connection_t 
 
     msg.msg_controllen = clen;
 
-    n = ngx_sendmsg(c, &msg, 0);
-    if (n < 0) {
-        return n;
+    for ( ;; ) {
+
+        preserved_pnum = ctx->pnum;
+
+        p = dst;
+
+        for (nmsg = 0; nmsg < UIO_MAXIOV; nmsg++) {
+
+            start = p;
+            end = p + NGX_QUIC_MAX_UDP_SEGMENT_BUF;
+
+            for (nseg = 0; nseg < NGX_QUIC_MAX_SEGMENTS; nseg++) {
+
+                if (cg->in_flight + (p - dst) >= cg->window) {
+                    break;
+                }
+
+                len = ngx_min(segsize, (size_t) (end - p));
+                if (len == 0) {
+                    break;
+                }
+
+                n = ngx_quic_output_packet(c, ctx, p, len, len);
+
+                if (n == NGX_ERROR) {
+                    return NGX_ERROR;
+                }
+
+                if (n == 0) {
+                    break;
+                }
+
+                p += n;
+            }
+
+            if (nseg == 0) {
+                break;
+            }
+
+            iovs[nmsg].iov_base = start;
+            iovs[nmsg].iov_len = p - start;
+
+            msgs[nmsg].msg_hdr = msg;
+            msgs[nmsg].msg_hdr.msg_iov = &iovs[nmsg];
+            msgs[nmsg].msg_hdr.msg_iovlen = 1;
+            msgs[nmsg].msg_len = 0;
+        }
+
+        if (nmsg == 0) {
+            break;
+        }
+
+    eintr:
+
+        n = sendmmsg(c->fd, msgs, nmsg, 0);
+
+        if (n == -1) {
+            err = ngx_errno;
+
+            switch (err) {
+            case NGX_EAGAIN:
+                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
+                               "sendmmsg() not ready");
+
+                ngx_quic_revert_send(c, ctx, preserved_pnum);
+
+                ngx_add_timer(&qc->push, NGX_QUIC_SOCKET_RETRY_DELAY);
+                return NGX_OK;
+
+            case NGX_EINTR:
+                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
+                               "sendmmsg() was interrupted");
+                goto eintr;
+
+            default:
+                c->write->error = 1;
+                ngx_connection_error(c, err, "sendmsg() failed");
+                return NGX_ERROR;
+            }
+        }
+
+        len = p - dst;
+
+        ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "sendmmsg: %z of %ui msg of size %uz", n, nmsg, len);
+
+        c->sent += len;
+
+        ngx_quic_commit_send(c, ctx);
+
+        path->sent += len;
     }
 
-    c->sent += n;
-
-    return n;
+    return NGX_OK;
 }
 
 #endif

From pluknet at nginx.com  Fri Jul 28 15:51:13 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Fri, 28 Jul 2023 19:51:13 +0400
Subject: [PATCH 0 of 4] QUIC path MTU discovery
In-Reply-To: <patchbomb.1688651833@arut-laptop>
References: <patchbomb.1680015097@arut-laptop>
 <patchbomb.1688651833@arut-laptop>
Message-ID: <85900FA7-5FBD-4AA8-AEF7-C74C635F9A84@nginx.com>


> On 6 Jul 2023, at 17:57, Roman Arutyunyan <arut at nginx.com> wrote:
> 
> Updated PMTUD:
> 
> - Eliminated getting local interface MTU.  This was too intrusive and platform-
>  dependent.

RFC 8899 how the following statement on this:

A DPLPMTUD sender is RECOMMENDED to utilize information about
the maximum size of packet that can be transmitted by the sender
on the local link (e.g., the local link MTU). <..> This avoids
implementations trying to send probe packets that cannot be
transferred by the local link.

I won't insist on keeping this part,
but we can reconsider it in future.

> 
> - Added timeout (1 sec) before each PMTUD iteration to reduce PMTUD traffic
>  in short connections.

While I understand that this is used to avoid over-utilising paths,
such long deferred probes can cause under-utilisation.
Probably some value derived from path's PTO might have sense, e.g. 3PTO.
The delay as implemented causes delays for subsequent probes as well,
which might be too conservative.

> 
> - Size search reimplemented.  MTU is increased by 256 bytes until failure,
>  after which binary search is used.  Real life MTU will be around 1500, while
>  initial payload size is 1200.  Doing binary search from 65k down to 1500 will
>  produce a lot of failed syscalls and useless traffic.  Ascending from 1200
>  to a first failre will be quick.

I agree that starting from below have more sense.
Further, if multiplying probe MTU, as proposed in addendum,
looks more suitable for benchmarking.
OTOH, 256 byte steps may be more realistic approach,
given that well known PMTUs usually don't exceed 1500 bytes.

> 
> --
> Roman Arutyunyan
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx-devel

-- 
Sergey Kandaurov

From pluknet at nginx.com  Fri Jul 28 15:51:16 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Fri, 28 Jul 2023 19:51:16 +0400
Subject: [PATCH 1 of 4] QUIC: removed path->limited flag
In-Reply-To: <df01e8582189126cd5de.1688651834@arut-laptop>
References: <patchbomb.1688651833@arut-laptop>
 <df01e8582189126cd5de.1688651834@arut-laptop>
Message-ID: <861BB30B-A608-4501-A63B-50E08101C49D@nginx.com>


> On 6 Jul 2023, at 17:57, Roman Arutyunyan <arut at nginx.com> wrote:
> 
> # HG changeset patch
> # User Roman Arutyunyan <arut at nginx.com>
> # Date 1688651341 -14400
> #      Thu Jul 06 17:49:01 2023 +0400
> # Node ID df01e8582189126cd5defd8f4bb021fec1a1a35a
> # Parent  77c1418916f7817a0d020f28d8a08f278a12fe43
> QUIC: removed path->limited flag.
> 
> Its value is the opposite of path->validated.
> 

Looks good.

-- 
Sergey Kandaurov

From pluknet at nginx.com  Fri Jul 28 15:51:18 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Fri, 28 Jul 2023 19:51:18 +0400
Subject: [PATCH 2 of 4] QUIC: removed explicit packet padding for certain
 frames
In-Reply-To: <f98ff9b62d2bccac10b6.1688651835@arut-laptop>
References: <patchbomb.1688651833@arut-laptop>
 <f98ff9b62d2bccac10b6.1688651835@arut-laptop>
Message-ID: <67346D5E-D13D-4C6D-AFDE-8BEE07B0F778@nginx.com>


> On 6 Jul 2023, at 17:57, Roman Arutyunyan <arut at nginx.com> wrote:
> 
> # HG changeset patch
> # User Roman Arutyunyan <arut at nginx.com>
> # Date 1688628647 -14400
> #      Thu Jul 06 11:30:47 2023 +0400
> # Node ID f98ff9b62d2bccac10b603b8183eca2ff0f4183e
> # Parent  df01e8582189126cd5defd8f4bb021fec1a1a35a
> QUIC: removed explicit packet padding for certain frames.
> 
> The frames for which the padding is removed are PATH_CHALLENGE and
> PATH_RESPONSE, which are sent separately by ngx_quic_frame_sendto().

Looks good.

We discussed this in the past, this code seems to be dead.

> 
> diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
> --- a/src/event/quic/ngx_event_quic_output.c
> +++ b/src/event/quic/ngx_event_quic_output.c
> @@ -525,7 +525,7 @@ ngx_quic_output_packet(ngx_connection_t 
>     ssize_t                 flen;
>     ngx_str_t               res;
>     ngx_int_t               rc;
> -    ngx_uint_t              nframes, expand;
> +    ngx_uint_t              nframes;
>     ngx_msec_t              now;
>     ngx_queue_t            *q;
>     ngx_quic_frame_t       *f;
> @@ -560,7 +560,6 @@ ngx_quic_output_packet(ngx_connection_t 
>     nframes = 0;
>     p = src;
>     len = 0;
> -    expand = 0;
> 
>     for (q = ngx_queue_head(&ctx->frames);
>          q != ngx_queue_sentinel(&ctx->frames);
> @@ -568,33 +567,6 @@ ngx_quic_output_packet(ngx_connection_t 
>     {
>         f = ngx_queue_data(q, ngx_quic_frame_t, queue);
> 
> -        if (!expand && (f->type == NGX_QUIC_FT_PATH_RESPONSE
> -                        || f->type == NGX_QUIC_FT_PATH_CHALLENGE))
> -        {
> -            /*
> -             * RFC 9000, 8.2.1.  Initiating Path Validation
> -             *
> -             * An endpoint MUST expand datagrams that contain a
> -             * PATH_CHALLENGE frame to at least the smallest allowed
> -             * maximum datagram size of 1200 bytes...
> -             *
> -             * (same applies to PATH_RESPONSE frames)
> -             */
> -
> -            if (max < 1200) {
> -                /* expanded packet will not fit */
> -                break;
> -            }
> -
> -            if (min < 1200) {
> -                min = 1200;
> -
> -                min_payload = ngx_quic_payload_size(&pkt, min);
> -            }
> -
> -            expand = 1;
> -        }
> -
>         if (len >= max_payload) {
>             break;
>         }
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx-devel

-- 
Sergey Kandaurov

From pluknet at nginx.com  Fri Jul 28 15:51:21 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Fri, 28 Jul 2023 19:51:21 +0400
Subject: [PATCH 3 of 4] QUIC: allowed ngx_quic_frame_sendto() to return
 NGX_AGAIN
In-Reply-To: <a1ea543d009311765144.1688651836@arut-laptop>
References: <patchbomb.1688651833@arut-laptop>
 <a1ea543d009311765144.1688651836@arut-laptop>
Message-ID: <BBB29018-AFD7-4D84-AB9C-383B6E7ED905@nginx.com>


> On 6 Jul 2023, at 17:57, Roman Arutyunyan <arut at nginx.com> wrote:
> 
> # HG changeset patch
> # User Roman Arutyunyan <arut at nginx.com>
> # Date 1679920727 -14400
> #      Mon Mar 27 16:38:47 2023 +0400
> # Node ID a1ea543d009311765144351b2557c00c8e6445bf
> # Parent  f98ff9b62d2bccac10b603b8183eca2ff0f4183e
> QUIC: allowed ngx_quic_frame_sendto() to return NGX_AGAIN.
> 
> Previously, NGX_AGAIN returned by ngx_send() was treated by
> ngx_quic_frame_sendto() as error, which triggered errors in its callers.
> However, a blocked socket is not an error.  Now NGX_AGAIN is passed as is to
> the ngx_quic_frame_sendto() callers, which can safely ignore it.

Looks good, as a prerequisite for PMTU probes.
Probably, it makes sense to reflect this in the description.

-- 
Sergey Kandaurov

From pluknet at nginx.com  Fri Jul 28 15:51:22 2023
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Fri, 28 Jul 2023 19:51:22 +0400
Subject: [PATCH 4 of 4] QUIC: path MTU discovery
In-Reply-To: <1a49fd5d2013b6c8d502.1688651837@arut-laptop>
References: <patchbomb.1688651833@arut-laptop>
 <1a49fd5d2013b6c8d502.1688651837@arut-laptop>
Message-ID: <E772A710-B4FC-4A53-8489-DEC5586B29E7@nginx.com>


> On 6 Jul 2023, at 17:57, Roman Arutyunyan <arut at nginx.com> wrote:
> 
> # HG changeset patch
> # User Roman Arutyunyan <arut at nginx.com>
> # Date 1688481216 -14400
> #      Tue Jul 04 18:33:36 2023 +0400
> # Node ID 1a49fd5d2013b6c8d50263499e6ced440927e949
> # Parent  a1ea543d009311765144351b2557c00c8e6445bf
> QUIC: path MTU discovery.
> 
> MTU selection starts by stepping up until the first failure.  Then binary
> search is used to find the path MTU.
> 
> diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
> --- a/src/core/ngx_connection.c
> +++ b/src/core/ngx_connection.c
> @@ -1583,6 +1583,10 @@ ngx_connection_error(ngx_connection_t *c
>     }
> #endif
> 
> +    if (err == NGX_EMSGSIZE && c->log_error == NGX_ERROR_IGNORE_EMSGSIZE) {
> +        return 0;
> +    }
> +
>     if (err == 0
>         || err == NGX_ECONNRESET
> #if (NGX_WIN32)
> @@ -1600,6 +1604,7 @@ ngx_connection_error(ngx_connection_t *c
>     {
>         switch (c->log_error) {
> 
> +        case NGX_ERROR_IGNORE_EMSGSIZE:
>         case NGX_ERROR_IGNORE_EINVAL:
>         case NGX_ERROR_IGNORE_ECONNRESET:
>         case NGX_ERROR_INFO:
> diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
> --- a/src/core/ngx_connection.h
> +++ b/src/core/ngx_connection.h
> @@ -97,7 +97,8 @@ typedef enum {
>     NGX_ERROR_ERR,
>     NGX_ERROR_INFO,
>     NGX_ERROR_IGNORE_ECONNRESET,
> -    NGX_ERROR_IGNORE_EINVAL
> +    NGX_ERROR_IGNORE_EINVAL,
> +    NGX_ERROR_IGNORE_EMSGSIZE
> } ngx_connection_log_error_e;
> 
> 
> diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
> --- a/src/event/quic/ngx_event_quic.c
> +++ b/src/event/quic/ngx_event_quic.c
> @@ -149,11 +149,6 @@ ngx_quic_apply_transport_params(ngx_conn
>         ngx_log_error(NGX_LOG_INFO, c->log, 0,
>                       "quic maximum packet size is invalid");
>         return NGX_ERROR;
> -
> -    } else if (ctp->max_udp_payload_size > ngx_quic_max_udp_payload(c)) {
> -        ctp->max_udp_payload_size = ngx_quic_max_udp_payload(c);
> -        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
> -                       "quic client maximum packet size truncated");
>     }
> 
>     if (ctp->active_connection_id_limit < 2) {
> @@ -286,7 +281,7 @@ ngx_quic_new_connection(ngx_connection_t
> 
>     qc->path_validation.log = c->log;
>     qc->path_validation.data = c;
> -    qc->path_validation.handler = ngx_quic_path_validation_handler;
> +    qc->path_validation.handler = ngx_quic_path_handler;
> 
>     qc->conf = conf;
> 
> @@ -297,7 +292,7 @@ ngx_quic_new_connection(ngx_connection_t
>     ctp = &qc->ctp;
> 
>     /* defaults to be used before actual client parameters are received */
> -    ctp->max_udp_payload_size = ngx_quic_max_udp_payload(c);
> +    ctp->max_udp_payload_size = NGX_QUIC_MAX_UDP_PAYLOAD_SIZE;
>     ctp->ack_delay_exponent = NGX_QUIC_DEFAULT_ACK_DELAY_EXPONENT;
>     ctp->max_ack_delay = NGX_QUIC_DEFAULT_MAX_ACK_DELAY;
>     ctp->active_connection_id_limit = 2;
> diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c
> --- a/src/event/quic/ngx_event_quic_ack.c
> +++ b/src/event/quic/ngx_event_quic_ack.c
> @@ -229,6 +229,12 @@ ngx_quic_handle_ack_frame_range(ngx_conn
> 
>     qc = ngx_quic_get_connection(c);
> 
> +    if (ctx->level == ssl_encryption_application) {
> +        if (ngx_quic_handle_path_mtu_ack(c, qc->path, min, max) != NGX_OK) {
> +            return NGX_ERROR;
> +        }
> +    }
> +
>     st->max_pn = NGX_TIMER_INFINITE;
>     found = 0;
> 
> diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h
> --- a/src/event/quic/ngx_event_quic_connection.h
> +++ b/src/event/quic/ngx_event_quic_connection.h
> @@ -66,6 +66,14 @@ typedef struct ngx_quic_keys_s        ng
> #define ngx_quic_get_socket(c)               ((ngx_quic_socket_t *)((c)->udp))
> 
> 
> +typedef enum {
> +    NGX_QUIC_PATH_IDLE = 0,
> +    NGX_QUIC_PATH_VALIDATING,
> +    NGX_QUIC_PATH_WAITING,
> +    NGX_QUIC_PATH_DISCOVERING_MTU

Nitpicking on too long name.
What about NGX_QUIC_PATH_MTUD or something.

> +} ngx_quic_path_state_e;
> +
> +
> struct ngx_quic_client_id_s {
>     ngx_queue_t                       queue;
>     uint64_t                          seqnum;
> @@ -89,18 +97,22 @@ struct ngx_quic_path_s {
>     ngx_sockaddr_t                    sa;
>     socklen_t                         socklen;
>     ngx_quic_client_id_t             *cid;
> +    ngx_quic_path_state_e             state;
>     ngx_msec_t                        expires;
>     ngx_uint_t                        tries;
>     ngx_uint_t                        tag;
> +    size_t                            mtu;
> +    size_t                            mtud;
> +    size_t                            max_mtu;
>     off_t                             sent;
>     off_t                             received;
>     u_char                            challenge1[8];
>     u_char                            challenge2[8];
>     uint64_t                          seqnum;
> +    uint64_t                          mtu_pnum[NGX_QUIC_PATH_RETRIES];
>     ngx_str_t                         addr_text;
>     u_char                            text[NGX_SOCKADDR_STRLEN];
> -    unsigned                          validated:1;
> -    unsigned                          validating:1;
> +    ngx_uint_t                        validated; /* unsigned validated:1; */
> };
> 
> 
> diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c
> --- a/src/event/quic/ngx_event_quic_migration.c
> +++ b/src/event/quic/ngx_event_quic_migration.c
> @@ -10,6 +10,11 @@
> #include <ngx_event_quic_connection.h>
> 
> 
> +#define NGX_QUIC_PATH_MTU_DELAY       1000

The value looks too much,
it would delay each subsequent probe (which is not retry).

> +#define NGX_QUIC_PATH_MTU_STEP        256
> +#define NGX_QUIC_PATH_MTU_PRECISION   16
> +
> +
> static void ngx_quic_set_connection_path(ngx_connection_t *c,
>     ngx_quic_path_t *path);
> static ngx_int_t ngx_quic_validate_path(ngx_connection_t *c,
> @@ -17,7 +22,15 @@ static ngx_int_t ngx_quic_validate_path(
> static ngx_int_t ngx_quic_send_path_challenge(ngx_connection_t *c,
>     ngx_quic_path_t *path);
> static void ngx_quic_set_path_timer(ngx_connection_t *c);
> +static ngx_int_t ngx_quic_expire_path_validation(ngx_connection_t *c,
> +    ngx_quic_path_t *path);
> +static ngx_int_t ngx_quic_expire_path_mtu_delay(ngx_connection_t *c,
> +    ngx_quic_path_t *path);
> +static ngx_int_t ngx_quic_expire_path_mtu_discovery(ngx_connection_t *c,
> +    ngx_quic_path_t *path);
> static ngx_quic_path_t *ngx_quic_get_path(ngx_connection_t *c, ngx_uint_t tag);
> +static ngx_int_t ngx_quic_send_path_mtu_probe(ngx_connection_t *c,
> +    ngx_quic_path_t *path);
> 
> 
> ngx_int_t
> @@ -97,7 +110,7 @@ ngx_quic_handle_path_response_frame(ngx_
>     {
>         path = ngx_queue_data(q, ngx_quic_path_t, queue);
> 
> -        if (!path->validating) {
> +        if (path->state != NGX_QUIC_PATH_VALIDATING) {
>             continue;
>         }
> 
> @@ -136,6 +149,9 @@ valid:
>         {
>             /* address did not change */
>             rst = 0;
> +
> +            path->mtu = prev->mtu;
> +            path->max_mtu = prev->max_mtu;
>         }
>     }
> 
> @@ -167,9 +183,8 @@ valid:
>     ngx_quic_path_dbg(c, "is validated", path);
> 
>     path->validated = 1;
> -    path->validating = 0;
> 
> -    ngx_quic_set_path_timer(c);
> +    ngx_quic_discover_path_mtu(c, path);
> 
>     return NGX_OK;
> }
> @@ -217,6 +232,8 @@ ngx_quic_new_path(ngx_connection_t *c,
>     path->addr_text.len = ngx_sock_ntop(sockaddr, socklen, path->text,
>                                         NGX_SOCKADDR_STRLEN, 1);
> 
> +    path->mtu = NGX_QUIC_MIN_INITIAL_SIZE;
> +
>     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
>                    "quic path seq:%uL created addr:%V",
>                    path->seqnum, &path->addr_text);
> @@ -464,7 +481,7 @@ ngx_quic_handle_migration(ngx_connection
> 
>     ngx_quic_set_connection_path(c, next);
> 
> -    if (!next->validated && !next->validating) {
> +    if (!next->validated && next->state != NGX_QUIC_PATH_VALIDATING) {
>         if (ngx_quic_validate_path(c, next) != NGX_OK) {
>             return NGX_ERROR;
>         }
> @@ -492,7 +509,6 @@ ngx_quic_validate_path(ngx_connection_t 
>     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
>                    "quic initiated validation of path seq:%uL", path->seqnum);
> 
> -    path->validating = 1;
>     path->tries = 0;
> 
>     if (RAND_bytes(path->challenge1, 8) != 1) {
> @@ -511,6 +527,7 @@ ngx_quic_validate_path(ngx_connection_t 
>     pto = ngx_max(ngx_quic_pto(c, ctx), 1000);
> 
>     path->expires = ngx_current_msec + pto;
> +    path->state = NGX_QUIC_PATH_VALIDATING;
> 
>     ngx_quic_set_path_timer(c);
> 
> @@ -558,6 +575,42 @@ ngx_quic_send_path_challenge(ngx_connect
> }
> 
> 
> +void
> +ngx_quic_discover_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path)
> +{
> +    ngx_quic_connection_t  *qc;
> +
> +    qc = ngx_quic_get_connection(c);
> +
> +    if (path->max_mtu) {
> +        if (path->max_mtu - path->mtu <= NGX_QUIC_PATH_MTU_PRECISION) {
> +            path->state = NGX_QUIC_PATH_IDLE;
> +            ngx_quic_set_path_timer(c);
> +            return;
> +        }
> +
> +        path->mtud = (path->mtu + path->max_mtu) / 2;
> +
> +    } else {
> +        path->mtud = path->mtu + NGX_QUIC_PATH_MTU_STEP;

I like this approach, compared to multiplying probe steps.
This better fits to the pattern to keep around a well-known
initial value, which is typically 1500 bytes.
Another one could be to let user specify its own initial
value for cases when he knows well its network environment.

> +
> +        if (path->mtud >= qc->ctp.max_udp_payload_size) {
> +            path->mtud = qc->ctp.max_udp_payload_size;
> +            path->max_mtu = qc->ctp.max_udp_payload_size;
> +        }
> +    }
> +
> +    path->state = NGX_QUIC_PATH_WAITING;
> +    path->expires = ngx_current_msec + NGX_QUIC_PATH_MTU_DELAY;
> +
> +    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
> +                   "quic path seq:%uL schedule mtu:%uz",
> +                   path->seqnum, path->mtud);
> +
> +    ngx_quic_set_path_timer(c);
> +}
> +
> +
> static void
> ngx_quic_set_path_timer(ngx_connection_t *c)
> {
> @@ -578,7 +631,7 @@ ngx_quic_set_path_timer(ngx_connection_t
>     {
>         path = ngx_queue_data(q, ngx_quic_path_t, queue);
> 
> -        if (!path->validating) {
> +        if (path->state == NGX_QUIC_PATH_IDLE) {
>             continue;
>         }
> 
> @@ -600,22 +653,18 @@ ngx_quic_set_path_timer(ngx_connection_t
> 
> 
> void
> -ngx_quic_path_validation_handler(ngx_event_t *ev)
> +ngx_quic_path_handler(ngx_event_t *ev)
> {
>     ngx_msec_t              now;
>     ngx_queue_t            *q;
> -    ngx_msec_int_t          left, next, pto;
> -    ngx_quic_path_t        *path, *bkp;
> +    ngx_msec_int_t          left;
> +    ngx_quic_path_t        *path;
>     ngx_connection_t       *c;
> -    ngx_quic_send_ctx_t    *ctx;
>     ngx_quic_connection_t  *qc;
> 
>     c = ev->data;
>     qc = ngx_quic_get_connection(c);
> 
> -    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> -
> -    next = -1;
>     now = ngx_current_msec;
> 
>     q = ngx_queue_head(&qc->paths);
> @@ -625,83 +674,274 @@ ngx_quic_path_validation_handler(ngx_eve
>         path = ngx_queue_data(q, ngx_quic_path_t, queue);
>         q = ngx_queue_next(q);
> 
> -        if (!path->validating) {
> +        if (path->state == NGX_QUIC_PATH_IDLE) {
>             continue;
>         }
> 
>         left = path->expires - now;
> 
>         if (left > 0) {
> -
> -            if (next == -1 || left < next) {
> -                next = left;
> -            }
> -
> -            continue;
> -        }
> -
> -        if (++path->tries < NGX_QUIC_PATH_RETRIES) {
> -            pto = ngx_max(ngx_quic_pto(c, ctx), 1000) << path->tries;
> -
> -            path->expires = ngx_current_msec + pto;
> -
> -            if (next == -1 || pto < next) {
> -                next = pto;
> -            }
> -
> -            /* retransmit */
> -            (void) ngx_quic_send_path_challenge(c, path);
> -
>             continue;
>         }
> 
> -        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
> -                       "quic path seq:%uL validation failed", path->seqnum);
> +        switch (path->state) {
> +        case NGX_QUIC_PATH_VALIDATING:
> +             if (ngx_quic_expire_path_validation(c, path) != NGX_OK) {
> +                goto failed;
> +            }
> +
> +            break;
> +
> +        case NGX_QUIC_PATH_WAITING:
> +            if (ngx_quic_expire_path_mtu_delay(c, path) != NGX_OK) {
> +                goto failed;
> +            }
> +
> +            break;
> +
> +        case NGX_QUIC_PATH_DISCOVERING_MTU:
> +            if (ngx_quic_expire_path_mtu_discovery(c, path) != NGX_OK) {
> +                goto failed;
> +             }
> +
> +            break;
> +

indentation in several places here

> +        default:
> +            break;
> +         }
> +     }
> +
> +    ngx_quic_set_path_timer(c);
> +
> +    return;
> 
> -        /* found expired path */
> +failed:
> +
> +    ngx_quic_close_connection(c, NGX_ERROR);
> +}
> +
> +
> +static ngx_int_t
> +ngx_quic_expire_path_validation(ngx_connection_t *c, ngx_quic_path_t *path)
> +{
> +    ngx_msec_int_t          pto;
> +    ngx_quic_path_t        *bkp;
> +    ngx_quic_send_ctx_t    *ctx;
> +    ngx_quic_connection_t  *qc;
> 
> -        path->validated = 0;
> -        path->validating = 0;
> +    qc = ngx_quic_get_connection(c);
> +    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> +
> +    if (++path->tries < NGX_QUIC_PATH_RETRIES) {
> +        pto = ngx_max(ngx_quic_pto(c, ctx), 1000) << path->tries;
> +        path->expires = ngx_current_msec + pto;
> +
> +        (void) ngx_quic_send_path_challenge(c, path);
> +
> +        return NGX_OK;
> +    }
> +
> +    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
> +                   "quic path seq:%uL validation failed", path->seqnum);
> +
> +    /* found expired path */
> +
> +    path->validated = 0;
> 
> 
> -        /* RFC 9000, 9.3.2.  On-Path Address Spoofing
> -         *
> -         * To protect the connection from failing due to such a spurious
> -         * migration, an endpoint MUST revert to using the last validated
> -         * peer address when validation of a new peer address fails.
> -         */
> -
> -        if (qc->path == path) {
> -            /* active path validation failed */
> -
> -            bkp = ngx_quic_get_path(c, NGX_QUIC_PATH_BACKUP);
> +    /* RFC 9000, 9.3.2.  On-Path Address Spoofing
> +     *
> +     * To protect the connection from failing due to such a spurious
> +     * migration, an endpoint MUST revert to using the last validated
> +     * peer address when validation of a new peer address fails.
> +     */
> 
> -            if (bkp == NULL) {
> -                qc->error = NGX_QUIC_ERR_NO_VIABLE_PATH;
> -                qc->error_reason = "no viable path";
> -                ngx_quic_close_connection(c, NGX_ERROR);
> -                return;
> -            }
> +    if (qc->path == path) {
> +        /* active path validation failed */
> +
> +        bkp = ngx_quic_get_path(c, NGX_QUIC_PATH_BACKUP);
> 
> -            qc->path = bkp;
> -            qc->path->tag = NGX_QUIC_PATH_ACTIVE;
> -
> -            ngx_quic_set_connection_path(c, qc->path);
> -
> -            ngx_log_error(NGX_LOG_INFO, c->log, 0,
> -                          "quic path seq:%uL addr:%V is restored from backup",
> -                          qc->path->seqnum, &qc->path->addr_text);
> -
> -            ngx_quic_path_dbg(c, "is active", qc->path);
> +        if (bkp == NULL) {
> +            qc->error = NGX_QUIC_ERR_NO_VIABLE_PATH;
> +            qc->error_reason = "no viable path";
> +            return NGX_ERROR;
>         }
> 
> -        if (ngx_quic_free_path(c, path) != NGX_OK) {
> -            ngx_quic_close_connection(c, NGX_ERROR);
> -            return;
> +        qc->path = bkp;
> +        qc->path->tag = NGX_QUIC_PATH_ACTIVE;
> +
> +        ngx_quic_set_connection_path(c, qc->path);
> +
> +        ngx_log_error(NGX_LOG_INFO, c->log, 0,
> +                      "quic path seq:%uL addr:%V is restored from backup",
> +                      qc->path->seqnum, &qc->path->addr_text);
> +
> +        ngx_quic_path_dbg(c, "is active", qc->path);
> +    }
> +
> +    return ngx_quic_free_path(c, path);
> +}
> +
> +
> +static ngx_int_t
> +ngx_quic_expire_path_mtu_delay(ngx_connection_t *c, ngx_quic_path_t *path)
> +{
> +    ngx_int_t               rc;
> +    ngx_uint_t              i;
> +    ngx_msec_t              pto;
> +    ngx_quic_send_ctx_t    *ctx;
> +    ngx_quic_connection_t  *qc;
> +
> +    qc = ngx_quic_get_connection(c);
> +    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> +
> +    path->tries = 0;
> +
> +    for ( ;; ) {
> +
> +        for (i = 0; i < NGX_QUIC_PATH_RETRIES; i++) {
> +            path->mtu_pnum[i] = NGX_QUIC_UNSET_PN;
> +        }
> +
> +        rc = ngx_quic_send_path_mtu_probe(c, path);
> +
> +        if (rc == NGX_ERROR) {
> +            return NGX_ERROR;
> +        }
> +
> +        if (rc == NGX_OK) {
> +            pto = ngx_quic_pto(c, ctx);
> +            path->expires = ngx_current_msec + pto;
> +            path->state = NGX_QUIC_PATH_DISCOVERING_MTU;
> +            return NGX_OK;
> +        }
> +
> +        /* rc == NGX_DECLINED */
> +
> +        path->max_mtu = path->mtud;
> +
> +        if (path->max_mtu - path->mtu <= NGX_QUIC_PATH_MTU_PRECISION) {
> +            path->state = NGX_QUIC_PATH_IDLE;
> +            return NGX_OK;
> +        }
> +
> +        path->mtud = (path->mtu + path->max_mtu) / 2;
> +    }
> +}
> +
> +
> +static ngx_int_t
> +ngx_quic_expire_path_mtu_discovery(ngx_connection_t *c, ngx_quic_path_t *path)
> +{
> +    ngx_int_t               rc;
> +    ngx_msec_int_t          pto;
> +    ngx_quic_send_ctx_t    *ctx;
> +    ngx_quic_connection_t  *qc;
> +
> +    qc = ngx_quic_get_connection(c);
> +    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> +
> +    if (++path->tries < NGX_QUIC_PATH_RETRIES) {
> +        pto = ngx_quic_pto(c, ctx) << path->tries;
> +        path->expires = ngx_current_msec + pto;
> +

Setting path->expires there seems useless.
It don't seem to be used in ngx_quic_send_path_mtu_probe(),
and ngx_quic_discover_path_mtu() used to reset path->expires.

> +        rc = ngx_quic_send_path_mtu_probe(c, path);
> +        if (rc != NGX_DECLINED) {
> +            return rc;
>         }
>     }
> 
> -    if (next != -1) {
> -        ngx_add_timer(&qc->path_validation, next);
> +    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
> +                   "quic path seq:%uL expired mtu:%uz",
> +                   path->seqnum, path->mtud);
> +
> +    path->max_mtu = path->mtud;
> +
> +    ngx_quic_discover_path_mtu(c, path);
> +
> +    return NGX_OK;
> +}
> +
> +
> +static ngx_int_t
> +ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path)
> +{
> +    ngx_int_t               rc;
> +    ngx_uint_t              log_error;
> +    ngx_quic_frame_t        frame;
> +    ngx_quic_send_ctx_t    *ctx;
> +    ngx_quic_connection_t  *qc;
> +
> +    ngx_memzero(&frame, sizeof(ngx_quic_frame_t));
> +
> +    frame.level = ssl_encryption_application;
> +    frame.type = NGX_QUIC_FT_PING;
> +
> +    qc = ngx_quic_get_connection(c);
> +    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> +    path->mtu_pnum[path->tries] = ctx->pnum;
> +
> +    ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
> +                   "quic path seq:%uL send probe "
> +                   "mtu:%uz pnum:%uL tries:%ui",
> +                   path->seqnum, path->mtud, ctx->pnum, path->tries);
> +
> +    log_error = c->log_error;
> +    c->log_error = NGX_ERROR_IGNORE_EMSGSIZE;
> +
> +    rc = ngx_quic_frame_sendto(c, &frame, path->mtud, path);
> +    c->log_error = log_error;
> +
> +    if (rc == NGX_ERROR) {
> +        if (c->write->error) {
> +            c->write->error = 0;
> +
> +            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
> +                           "quic path seq:%uL rejected mtu:%uz",
> +                           path->seqnum, path->mtud);
> +
> +            return NGX_DECLINED;

The error handling looks awkward to decline on write error.
ngx_sendmsg() sets write error on fatal sendmsg errors,
and keeps it unset NGX_EAGAIN and NGX_EINTR, which
are typically non-fatal transient errors.
I understand this is done mostly to ignore EMSGSIZE,
maybe it makes sense to ignore the rest sendmsg() errors.

> +        }
> +
> +        return NGX_ERROR;
>     }
> +
> +    return NGX_OK;
> }
> +
> +
> +ngx_int_t
> +ngx_quic_handle_path_mtu_ack(ngx_connection_t *c, ngx_quic_path_t *path,
> +    uint64_t min, uint64_t max)
> +{
> +    uint64_t    pnum;
> +    ngx_uint_t  i;
> +
> +    if (path->state != NGX_QUIC_PATH_DISCOVERING_MTU) {
> +        return NGX_OK;
> +    }
> +
> +    for (i = 0; i < NGX_QUIC_PATH_RETRIES; i++) {
> +        pnum = path->mtu_pnum[i];
> +
> +        if (pnum == NGX_QUIC_UNSET_PN) {
> +            break;
> +        }
> +
> +        if (pnum < min || pnum > max) {
> +            continue;
> +        }
> +
> +        path->mtu = path->mtud;
> +
> +        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
> +                       "quic path seq:%uL ack mtu:%uz",
> +                       path->seqnum, path->mtu);
> +
> +        ngx_quic_discover_path_mtu(c, path);
> +
> +        break;

What if two probes appeared ACKed in one range.
Though that seems to be safe to break there, as two
probes barely have different probe sizes, rather retries.

> +    }
> +
> +    return NGX_OK;
> +}
> diff --git a/src/event/quic/ngx_event_quic_migration.h b/src/event/quic/ngx_event_quic_migration.h
> --- a/src/event/quic/ngx_event_quic_migration.h
> +++ b/src/event/quic/ngx_event_quic_migration.h
> @@ -18,10 +18,10 @@
> #define NGX_QUIC_PATH_BACKUP    2
> 
> #define ngx_quic_path_dbg(c, msg, path)                                       \
> -    ngx_log_debug6(NGX_LOG_DEBUG_EVENT, c->log, 0,                            \
> -                   "quic path seq:%uL %s sent:%O recvd:%O state:%s%s",        \
> +    ngx_log_debug7(NGX_LOG_DEBUG_EVENT, c->log, 0,                            \
> +                   "quic path seq:%uL %s tx:%O rx:%O valid:%d st:%d mtu:%uz", \
>                    path->seqnum, msg, path->sent, path->received,             \
> -                   path->validated ? "V": "N", path->validating ? "R": "");
> +                   path->validated, path->state, path->mtu);
> 
> ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c,
>     ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f);
> @@ -36,6 +36,10 @@ ngx_int_t ngx_quic_set_path(ngx_connecti
> ngx_int_t ngx_quic_handle_migration(ngx_connection_t *c,
>     ngx_quic_header_t *pkt);
> 
> -void ngx_quic_path_validation_handler(ngx_event_t *ev);
> +void ngx_quic_path_handler(ngx_event_t *ev);
> +
> +void ngx_quic_discover_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path);
> +ngx_int_t ngx_quic_handle_path_mtu_ack(ngx_connection_t *c,
> +    ngx_quic_path_t *path, uint64_t min, uint64_t max);

Nitpicking:
maybe just ngx_quic_handle_path_mtu() ?
This would match ngx_quic_discover_path_mtu() naming.

> 
> #endif /* _NGX_EVENT_QUIC_MIGRATION_H_INCLUDED_ */
> diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
> --- a/src/event/quic/ngx_event_quic_output.c
> +++ b/src/event/quic/ngx_event_quic_output.c
> @@ -10,9 +10,6 @@
> #include <ngx_event_quic_connection.h>
> 
> 
> -#define NGX_QUIC_MAX_UDP_PAYLOAD_OUT   1252
> -#define NGX_QUIC_MAX_UDP_PAYLOAD_OUT6  1232
> -
> #define NGX_QUIC_MAX_UDP_SEGMENT_BUF  65487 /* 65K - IPv6 header */
> #define NGX_QUIC_MAX_SEGMENTS            64 /* UDP_MAX_SEGMENTS */
> 
> @@ -61,21 +58,6 @@ static size_t ngx_quic_path_limit(ngx_co
>     size_t size);
> 
> 
> -size_t
> -ngx_quic_max_udp_payload(ngx_connection_t *c)
> -{
> -    /* TODO: path MTU discovery */
> -
> -#if (NGX_HAVE_INET6)
> -    if (c->sockaddr->sa_family == AF_INET6) {
> -        return NGX_QUIC_MAX_UDP_PAYLOAD_OUT6;
> -    }
> -#endif
> -
> -    return NGX_QUIC_MAX_UDP_PAYLOAD_OUT;
> -}
> -
> -
> ngx_int_t
> ngx_quic_output(ngx_connection_t *c)
> {
> @@ -142,10 +124,7 @@ ngx_quic_create_datagrams(ngx_connection
> 
>         p = dst;
> 
> -        len = ngx_min(qc->ctp.max_udp_payload_size,
> -                      NGX_QUIC_MAX_UDP_PAYLOAD_SIZE);
> -
> -        len = ngx_quic_path_limit(c, path, len);
> +        len = ngx_quic_path_limit(c, path, path->mtu);
> 
>         pad = ngx_quic_get_padding_level(c);
> 
> @@ -299,9 +278,7 @@ ngx_quic_allow_segmentation(ngx_connecti
>     ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> 
>     bytes = 0;
> -
> -    len = ngx_min(qc->ctp.max_udp_payload_size,
> -                  NGX_QUIC_MAX_UDP_SEGMENT_BUF);
> +    len = ngx_min(qc->path->mtu, NGX_QUIC_MAX_UDP_SEGMENT_BUF);
> 
>     for (q = ngx_queue_head(&ctx->frames);
>          q != ngx_queue_sentinel(&ctx->frames);
> @@ -345,8 +322,7 @@ ngx_quic_create_segments(ngx_connection_
>         return NGX_ERROR;
>     }
> 
> -    segsize = ngx_min(qc->ctp.max_udp_payload_size,
> -                      NGX_QUIC_MAX_UDP_SEGMENT_BUF);
> +    segsize = ngx_min(path->mtu, NGX_QUIC_MAX_UDP_SEGMENT_BUF);
>     p = dst;
>     end = dst + sizeof(dst);
> 
> diff --git a/src/event/quic/ngx_event_quic_output.h b/src/event/quic/ngx_event_quic_output.h
> --- a/src/event/quic/ngx_event_quic_output.h
> +++ b/src/event/quic/ngx_event_quic_output.h
> @@ -12,8 +12,6 @@
> #include <ngx_core.h>
> 
> 
> -size_t ngx_quic_max_udp_payload(ngx_connection_t *c);
> -
> ngx_int_t ngx_quic_output(ngx_connection_t *c);
> 
> ngx_int_t ngx_quic_negotiate_version(ngx_connection_t *c,
> diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c
> --- a/src/event/quic/ngx_event_quic_ssl.c
> +++ b/src/event/quic/ngx_event_quic_ssl.c
> @@ -494,6 +494,8 @@ ngx_quic_crypto_input(ngx_connection_t *
>      */
>     ngx_quic_discard_ctx(c, ssl_encryption_handshake);
> 
> +    ngx_quic_discover_path_mtu(c, qc->path);
> +
>     /* start accepting clients on negotiated number of server ids */
>     if (ngx_quic_create_sockets(c) != NGX_OK) {
>         return NGX_ERROR;
> diff --git a/src/os/unix/ngx_errno.h b/src/os/unix/ngx_errno.h
> --- a/src/os/unix/ngx_errno.h
> +++ b/src/os/unix/ngx_errno.h
> @@ -54,6 +54,7 @@ typedef int               ngx_err_t;
> #define NGX_ENOMOREFILES  0
> #define NGX_ELOOP         ELOOP
> #define NGX_EBADF         EBADF
> +#define NGX_EMSGSIZE      EMSGSIZE
> 
> #if (NGX_HAVE_OPENAT)
> #define NGX_EMLINK        EMLINK
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx-devel

-- 
Sergey Kandaurov

From arut at nginx.com  Mon Jul 31 17:34:20 2023
From: arut at nginx.com (Roman Arutyunyan)
Date: Mon, 31 Jul 2023 21:34:20 +0400
Subject: [PATCH 4 of 4] QUIC: path MTU discovery
In-Reply-To: <E772A710-B4FC-4A53-8489-DEC5586B29E7@nginx.com>
References: <patchbomb.1688651833@arut-laptop>
 <1a49fd5d2013b6c8d502.1688651837@arut-laptop>
 <E772A710-B4FC-4A53-8489-DEC5586B29E7@nginx.com>
Message-ID: <20230731173420.v7dq7bktey2ox4s2@N00W24XTQX>

Hi,

On Fri, Jul 28, 2023 at 07:51:22PM +0400, Sergey Kandaurov wrote:
> 
> > On 6 Jul 2023, at 17:57, Roman Arutyunyan <arut at nginx.com> wrote:
> > 
> > # HG changeset patch
> > # User Roman Arutyunyan <arut at nginx.com>
> > # Date 1688481216 -14400
> > #      Tue Jul 04 18:33:36 2023 +0400
> > # Node ID 1a49fd5d2013b6c8d50263499e6ced440927e949
> > # Parent  a1ea543d009311765144351b2557c00c8e6445bf
> > QUIC: path MTU discovery.
> > 
> > MTU selection starts by stepping up until the first failure.  Then binary
> > search is used to find the path MTU.
> > 
> > diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
> > --- a/src/core/ngx_connection.c
> > +++ b/src/core/ngx_connection.c
> > @@ -1583,6 +1583,10 @@ ngx_connection_error(ngx_connection_t *c
> >     }
> > #endif
> > 
> > +    if (err == NGX_EMSGSIZE && c->log_error == NGX_ERROR_IGNORE_EMSGSIZE) {
> > +        return 0;
> > +    }
> > +
> >     if (err == 0
> >         || err == NGX_ECONNRESET
> > #if (NGX_WIN32)
> > @@ -1600,6 +1604,7 @@ ngx_connection_error(ngx_connection_t *c
> >     {
> >         switch (c->log_error) {
> > 
> > +        case NGX_ERROR_IGNORE_EMSGSIZE:
> >         case NGX_ERROR_IGNORE_EINVAL:
> >         case NGX_ERROR_IGNORE_ECONNRESET:
> >         case NGX_ERROR_INFO:
> > diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
> > --- a/src/core/ngx_connection.h
> > +++ b/src/core/ngx_connection.h
> > @@ -97,7 +97,8 @@ typedef enum {
> >     NGX_ERROR_ERR,
> >     NGX_ERROR_INFO,
> >     NGX_ERROR_IGNORE_ECONNRESET,
> > -    NGX_ERROR_IGNORE_EINVAL
> > +    NGX_ERROR_IGNORE_EINVAL,
> > +    NGX_ERROR_IGNORE_EMSGSIZE
> > } ngx_connection_log_error_e;
> > 
> > 
> > diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
> > --- a/src/event/quic/ngx_event_quic.c
> > +++ b/src/event/quic/ngx_event_quic.c
> > @@ -149,11 +149,6 @@ ngx_quic_apply_transport_params(ngx_conn
> >         ngx_log_error(NGX_LOG_INFO, c->log, 0,
> >                       "quic maximum packet size is invalid");
> >         return NGX_ERROR;
> > -
> > -    } else if (ctp->max_udp_payload_size > ngx_quic_max_udp_payload(c)) {
> > -        ctp->max_udp_payload_size = ngx_quic_max_udp_payload(c);
> > -        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
> > -                       "quic client maximum packet size truncated");
> >     }
> > 
> >     if (ctp->active_connection_id_limit < 2) {
> > @@ -286,7 +281,7 @@ ngx_quic_new_connection(ngx_connection_t
> > 
> >     qc->path_validation.log = c->log;
> >     qc->path_validation.data = c;
> > -    qc->path_validation.handler = ngx_quic_path_validation_handler;
> > +    qc->path_validation.handler = ngx_quic_path_handler;
> > 
> >     qc->conf = conf;
> > 
> > @@ -297,7 +292,7 @@ ngx_quic_new_connection(ngx_connection_t
> >     ctp = &qc->ctp;
> > 
> >     /* defaults to be used before actual client parameters are received */
> > -    ctp->max_udp_payload_size = ngx_quic_max_udp_payload(c);
> > +    ctp->max_udp_payload_size = NGX_QUIC_MAX_UDP_PAYLOAD_SIZE;
> >     ctp->ack_delay_exponent = NGX_QUIC_DEFAULT_ACK_DELAY_EXPONENT;
> >     ctp->max_ack_delay = NGX_QUIC_DEFAULT_MAX_ACK_DELAY;
> >     ctp->active_connection_id_limit = 2;
> > diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c
> > --- a/src/event/quic/ngx_event_quic_ack.c
> > +++ b/src/event/quic/ngx_event_quic_ack.c
> > @@ -229,6 +229,12 @@ ngx_quic_handle_ack_frame_range(ngx_conn
> > 
> >     qc = ngx_quic_get_connection(c);
> > 
> > +    if (ctx->level == ssl_encryption_application) {
> > +        if (ngx_quic_handle_path_mtu_ack(c, qc->path, min, max) != NGX_OK) {
> > +            return NGX_ERROR;
> > +        }
> > +    }
> > +
> >     st->max_pn = NGX_TIMER_INFINITE;
> >     found = 0;
> > 
> > diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h
> > --- a/src/event/quic/ngx_event_quic_connection.h
> > +++ b/src/event/quic/ngx_event_quic_connection.h
> > @@ -66,6 +66,14 @@ typedef struct ngx_quic_keys_s        ng
> > #define ngx_quic_get_socket(c)               ((ngx_quic_socket_t *)((c)->udp))
> > 
> > 
> > +typedef enum {
> > +    NGX_QUIC_PATH_IDLE = 0,
> > +    NGX_QUIC_PATH_VALIDATING,
> > +    NGX_QUIC_PATH_WAITING,
> > +    NGX_QUIC_PATH_DISCOVERING_MTU
> 
> Nitpicking on too long name.
> What about NGX_QUIC_PATH_MTUD or something.

OK, renamed.

> > +} ngx_quic_path_state_e;
> > +
> > +
> > struct ngx_quic_client_id_s {
> >     ngx_queue_t                       queue;
> >     uint64_t                          seqnum;
> > @@ -89,18 +97,22 @@ struct ngx_quic_path_s {
> >     ngx_sockaddr_t                    sa;
> >     socklen_t                         socklen;
> >     ngx_quic_client_id_t             *cid;
> > +    ngx_quic_path_state_e             state;
> >     ngx_msec_t                        expires;
> >     ngx_uint_t                        tries;
> >     ngx_uint_t                        tag;
> > +    size_t                            mtu;
> > +    size_t                            mtud;
> > +    size_t                            max_mtu;
> >     off_t                             sent;
> >     off_t                             received;
> >     u_char                            challenge1[8];
> >     u_char                            challenge2[8];
> >     uint64_t                          seqnum;
> > +    uint64_t                          mtu_pnum[NGX_QUIC_PATH_RETRIES];
> >     ngx_str_t                         addr_text;
> >     u_char                            text[NGX_SOCKADDR_STRLEN];
> > -    unsigned                          validated:1;
> > -    unsigned                          validating:1;
> > +    ngx_uint_t                        validated; /* unsigned validated:1; */
> > };
> > 
> > 
> > diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c
> > --- a/src/event/quic/ngx_event_quic_migration.c
> > +++ b/src/event/quic/ngx_event_quic_migration.c
> > @@ -10,6 +10,11 @@
> > #include <ngx_event_quic_connection.h>
> > 
> > 
> > +#define NGX_QUIC_PATH_MTU_DELAY       1000
> 
> The value looks too much,
> it would delay each subsequent probe (which is not retry).

Probably the right value should be determined in real-life scenarios.
Let's decrease the value by a factor of 10 for now.

> > +#define NGX_QUIC_PATH_MTU_STEP        256
> > +#define NGX_QUIC_PATH_MTU_PRECISION   16
> > +
> > +
> > static void ngx_quic_set_connection_path(ngx_connection_t *c,
> >     ngx_quic_path_t *path);
> > static ngx_int_t ngx_quic_validate_path(ngx_connection_t *c,
> > @@ -17,7 +22,15 @@ static ngx_int_t ngx_quic_validate_path(
> > static ngx_int_t ngx_quic_send_path_challenge(ngx_connection_t *c,
> >     ngx_quic_path_t *path);
> > static void ngx_quic_set_path_timer(ngx_connection_t *c);
> > +static ngx_int_t ngx_quic_expire_path_validation(ngx_connection_t *c,
> > +    ngx_quic_path_t *path);
> > +static ngx_int_t ngx_quic_expire_path_mtu_delay(ngx_connection_t *c,
> > +    ngx_quic_path_t *path);
> > +static ngx_int_t ngx_quic_expire_path_mtu_discovery(ngx_connection_t *c,
> > +    ngx_quic_path_t *path);
> > static ngx_quic_path_t *ngx_quic_get_path(ngx_connection_t *c, ngx_uint_t tag);
> > +static ngx_int_t ngx_quic_send_path_mtu_probe(ngx_connection_t *c,
> > +    ngx_quic_path_t *path);
> > 
> > 
> > ngx_int_t
> > @@ -97,7 +110,7 @@ ngx_quic_handle_path_response_frame(ngx_
> >     {
> >         path = ngx_queue_data(q, ngx_quic_path_t, queue);
> > 
> > -        if (!path->validating) {
> > +        if (path->state != NGX_QUIC_PATH_VALIDATING) {
> >             continue;
> >         }
> > 
> > @@ -136,6 +149,9 @@ valid:
> >         {
> >             /* address did not change */
> >             rst = 0;
> > +
> > +            path->mtu = prev->mtu;
> > +            path->max_mtu = prev->max_mtu;
> >         }
> >     }
> > 
> > @@ -167,9 +183,8 @@ valid:
> >     ngx_quic_path_dbg(c, "is validated", path);
> > 
> >     path->validated = 1;
> > -    path->validating = 0;
> > 
> > -    ngx_quic_set_path_timer(c);
> > +    ngx_quic_discover_path_mtu(c, path);
> > 
> >     return NGX_OK;
> > }
> > @@ -217,6 +232,8 @@ ngx_quic_new_path(ngx_connection_t *c,
> >     path->addr_text.len = ngx_sock_ntop(sockaddr, socklen, path->text,
> >                                         NGX_SOCKADDR_STRLEN, 1);
> > 
> > +    path->mtu = NGX_QUIC_MIN_INITIAL_SIZE;
> > +
> >     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
> >                    "quic path seq:%uL created addr:%V",
> >                    path->seqnum, &path->addr_text);
> > @@ -464,7 +481,7 @@ ngx_quic_handle_migration(ngx_connection
> > 
> >     ngx_quic_set_connection_path(c, next);
> > 
> > -    if (!next->validated && !next->validating) {
> > +    if (!next->validated && next->state != NGX_QUIC_PATH_VALIDATING) {
> >         if (ngx_quic_validate_path(c, next) != NGX_OK) {
> >             return NGX_ERROR;
> >         }
> > @@ -492,7 +509,6 @@ ngx_quic_validate_path(ngx_connection_t 
> >     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
> >                    "quic initiated validation of path seq:%uL", path->seqnum);
> > 
> > -    path->validating = 1;
> >     path->tries = 0;
> > 
> >     if (RAND_bytes(path->challenge1, 8) != 1) {
> > @@ -511,6 +527,7 @@ ngx_quic_validate_path(ngx_connection_t 
> >     pto = ngx_max(ngx_quic_pto(c, ctx), 1000);
> > 
> >     path->expires = ngx_current_msec + pto;
> > +    path->state = NGX_QUIC_PATH_VALIDATING;
> > 
> >     ngx_quic_set_path_timer(c);
> > 
> > @@ -558,6 +575,42 @@ ngx_quic_send_path_challenge(ngx_connect
> > }
> > 
> > 
> > +void
> > +ngx_quic_discover_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path)
> > +{
> > +    ngx_quic_connection_t  *qc;
> > +
> > +    qc = ngx_quic_get_connection(c);
> > +
> > +    if (path->max_mtu) {
> > +        if (path->max_mtu - path->mtu <= NGX_QUIC_PATH_MTU_PRECISION) {
> > +            path->state = NGX_QUIC_PATH_IDLE;
> > +            ngx_quic_set_path_timer(c);
> > +            return;
> > +        }
> > +
> > +        path->mtud = (path->mtu + path->max_mtu) / 2;
> > +
> > +    } else {
> > +        path->mtud = path->mtu + NGX_QUIC_PATH_MTU_STEP;
> 
> I like this approach, compared to multiplying probe steps.
> This better fits to the pattern to keep around a well-known
> initial value, which is typically 1500 bytes.
> Another one could be to let user specify its own initial
> value for cases when he knows well its network environment.
> 
> > +
> > +        if (path->mtud >= qc->ctp.max_udp_payload_size) {
> > +            path->mtud = qc->ctp.max_udp_payload_size;
> > +            path->max_mtu = qc->ctp.max_udp_payload_size;
> > +        }
> > +    }
> > +
> > +    path->state = NGX_QUIC_PATH_WAITING;
> > +    path->expires = ngx_current_msec + NGX_QUIC_PATH_MTU_DELAY;
> > +
> > +    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
> > +                   "quic path seq:%uL schedule mtu:%uz",
> > +                   path->seqnum, path->mtud);
> > +
> > +    ngx_quic_set_path_timer(c);
> > +}
> > +
> > +
> > static void
> > ngx_quic_set_path_timer(ngx_connection_t *c)
> > {
> > @@ -578,7 +631,7 @@ ngx_quic_set_path_timer(ngx_connection_t
> >     {
> >         path = ngx_queue_data(q, ngx_quic_path_t, queue);
> > 
> > -        if (!path->validating) {
> > +        if (path->state == NGX_QUIC_PATH_IDLE) {
> >             continue;
> >         }
> > 
> > @@ -600,22 +653,18 @@ ngx_quic_set_path_timer(ngx_connection_t
> > 
> > 
> > void
> > -ngx_quic_path_validation_handler(ngx_event_t *ev)
> > +ngx_quic_path_handler(ngx_event_t *ev)
> > {
> >     ngx_msec_t              now;
> >     ngx_queue_t            *q;
> > -    ngx_msec_int_t          left, next, pto;
> > -    ngx_quic_path_t        *path, *bkp;
> > +    ngx_msec_int_t          left;
> > +    ngx_quic_path_t        *path;
> >     ngx_connection_t       *c;
> > -    ngx_quic_send_ctx_t    *ctx;
> >     ngx_quic_connection_t  *qc;
> > 
> >     c = ev->data;
> >     qc = ngx_quic_get_connection(c);
> > 
> > -    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> > -
> > -    next = -1;
> >     now = ngx_current_msec;
> > 
> >     q = ngx_queue_head(&qc->paths);
> > @@ -625,83 +674,274 @@ ngx_quic_path_validation_handler(ngx_eve
> >         path = ngx_queue_data(q, ngx_quic_path_t, queue);
> >         q = ngx_queue_next(q);
> > 
> > -        if (!path->validating) {
> > +        if (path->state == NGX_QUIC_PATH_IDLE) {
> >             continue;
> >         }
> > 
> >         left = path->expires - now;
> > 
> >         if (left > 0) {
> > -
> > -            if (next == -1 || left < next) {
> > -                next = left;
> > -            }
> > -
> > -            continue;
> > -        }
> > -
> > -        if (++path->tries < NGX_QUIC_PATH_RETRIES) {
> > -            pto = ngx_max(ngx_quic_pto(c, ctx), 1000) << path->tries;
> > -
> > -            path->expires = ngx_current_msec + pto;
> > -
> > -            if (next == -1 || pto < next) {
> > -                next = pto;
> > -            }
> > -
> > -            /* retransmit */
> > -            (void) ngx_quic_send_path_challenge(c, path);
> > -
> >             continue;
> >         }
> > 
> > -        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
> > -                       "quic path seq:%uL validation failed", path->seqnum);
> > +        switch (path->state) {
> > +        case NGX_QUIC_PATH_VALIDATING:
> > +             if (ngx_quic_expire_path_validation(c, path) != NGX_OK) {
> > +                goto failed;
> > +            }
> > +
> > +            break;
> > +
> > +        case NGX_QUIC_PATH_WAITING:
> > +            if (ngx_quic_expire_path_mtu_delay(c, path) != NGX_OK) {
> > +                goto failed;
> > +            }
> > +
> > +            break;
> > +
> > +        case NGX_QUIC_PATH_DISCOVERING_MTU:
> > +            if (ngx_quic_expire_path_mtu_discovery(c, path) != NGX_OK) {
> > +                goto failed;
> > +             }
> > +
> > +            break;
> > +
> 
> indentation in several places here

Fixed, thanks.

> > +        default:
> > +            break;
> > +         }
> > +     }
> > +
> > +    ngx_quic_set_path_timer(c);
> > +
> > +    return;
> > 
> > -        /* found expired path */
> > +failed:
> > +
> > +    ngx_quic_close_connection(c, NGX_ERROR);
> > +}
> > +
> > +
> > +static ngx_int_t
> > +ngx_quic_expire_path_validation(ngx_connection_t *c, ngx_quic_path_t *path)
> > +{
> > +    ngx_msec_int_t          pto;
> > +    ngx_quic_path_t        *bkp;
> > +    ngx_quic_send_ctx_t    *ctx;
> > +    ngx_quic_connection_t  *qc;
> > 
> > -        path->validated = 0;
> > -        path->validating = 0;
> > +    qc = ngx_quic_get_connection(c);
> > +    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> > +
> > +    if (++path->tries < NGX_QUIC_PATH_RETRIES) {
> > +        pto = ngx_max(ngx_quic_pto(c, ctx), 1000) << path->tries;
> > +        path->expires = ngx_current_msec + pto;
> > +
> > +        (void) ngx_quic_send_path_challenge(c, path);
> > +
> > +        return NGX_OK;
> > +    }
> > +
> > +    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
> > +                   "quic path seq:%uL validation failed", path->seqnum);
> > +
> > +    /* found expired path */
> > +
> > +    path->validated = 0;
> > 
> > 
> > -        /* RFC 9000, 9.3.2.  On-Path Address Spoofing
> > -         *
> > -         * To protect the connection from failing due to such a spurious
> > -         * migration, an endpoint MUST revert to using the last validated
> > -         * peer address when validation of a new peer address fails.
> > -         */
> > -
> > -        if (qc->path == path) {
> > -            /* active path validation failed */
> > -
> > -            bkp = ngx_quic_get_path(c, NGX_QUIC_PATH_BACKUP);
> > +    /* RFC 9000, 9.3.2.  On-Path Address Spoofing
> > +     *
> > +     * To protect the connection from failing due to such a spurious
> > +     * migration, an endpoint MUST revert to using the last validated
> > +     * peer address when validation of a new peer address fails.
> > +     */
> > 
> > -            if (bkp == NULL) {
> > -                qc->error = NGX_QUIC_ERR_NO_VIABLE_PATH;
> > -                qc->error_reason = "no viable path";
> > -                ngx_quic_close_connection(c, NGX_ERROR);
> > -                return;
> > -            }
> > +    if (qc->path == path) {
> > +        /* active path validation failed */
> > +
> > +        bkp = ngx_quic_get_path(c, NGX_QUIC_PATH_BACKUP);
> > 
> > -            qc->path = bkp;
> > -            qc->path->tag = NGX_QUIC_PATH_ACTIVE;
> > -
> > -            ngx_quic_set_connection_path(c, qc->path);
> > -
> > -            ngx_log_error(NGX_LOG_INFO, c->log, 0,
> > -                          "quic path seq:%uL addr:%V is restored from backup",
> > -                          qc->path->seqnum, &qc->path->addr_text);
> > -
> > -            ngx_quic_path_dbg(c, "is active", qc->path);
> > +        if (bkp == NULL) {
> > +            qc->error = NGX_QUIC_ERR_NO_VIABLE_PATH;
> > +            qc->error_reason = "no viable path";
> > +            return NGX_ERROR;
> >         }
> > 
> > -        if (ngx_quic_free_path(c, path) != NGX_OK) {
> > -            ngx_quic_close_connection(c, NGX_ERROR);
> > -            return;
> > +        qc->path = bkp;
> > +        qc->path->tag = NGX_QUIC_PATH_ACTIVE;
> > +
> > +        ngx_quic_set_connection_path(c, qc->path);
> > +
> > +        ngx_log_error(NGX_LOG_INFO, c->log, 0,
> > +                      "quic path seq:%uL addr:%V is restored from backup",
> > +                      qc->path->seqnum, &qc->path->addr_text);
> > +
> > +        ngx_quic_path_dbg(c, "is active", qc->path);
> > +    }
> > +
> > +    return ngx_quic_free_path(c, path);
> > +}
> > +
> > +
> > +static ngx_int_t
> > +ngx_quic_expire_path_mtu_delay(ngx_connection_t *c, ngx_quic_path_t *path)
> > +{
> > +    ngx_int_t               rc;
> > +    ngx_uint_t              i;
> > +    ngx_msec_t              pto;
> > +    ngx_quic_send_ctx_t    *ctx;
> > +    ngx_quic_connection_t  *qc;
> > +
> > +    qc = ngx_quic_get_connection(c);
> > +    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> > +
> > +    path->tries = 0;
> > +
> > +    for ( ;; ) {
> > +
> > +        for (i = 0; i < NGX_QUIC_PATH_RETRIES; i++) {
> > +            path->mtu_pnum[i] = NGX_QUIC_UNSET_PN;
> > +        }
> > +
> > +        rc = ngx_quic_send_path_mtu_probe(c, path);
> > +
> > +        if (rc == NGX_ERROR) {
> > +            return NGX_ERROR;
> > +        }
> > +
> > +        if (rc == NGX_OK) {
> > +            pto = ngx_quic_pto(c, ctx);
> > +            path->expires = ngx_current_msec + pto;
> > +            path->state = NGX_QUIC_PATH_DISCOVERING_MTU;
> > +            return NGX_OK;
> > +        }
> > +
> > +        /* rc == NGX_DECLINED */
> > +
> > +        path->max_mtu = path->mtud;
> > +
> > +        if (path->max_mtu - path->mtu <= NGX_QUIC_PATH_MTU_PRECISION) {
> > +            path->state = NGX_QUIC_PATH_IDLE;
> > +            return NGX_OK;
> > +        }
> > +
> > +        path->mtud = (path->mtu + path->max_mtu) / 2;
> > +    }
> > +}
> > +
> > +
> > +static ngx_int_t
> > +ngx_quic_expire_path_mtu_discovery(ngx_connection_t *c, ngx_quic_path_t *path)
> > +{
> > +    ngx_int_t               rc;
> > +    ngx_msec_int_t          pto;
> > +    ngx_quic_send_ctx_t    *ctx;
> > +    ngx_quic_connection_t  *qc;
> > +
> > +    qc = ngx_quic_get_connection(c);
> > +    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> > +
> > +    if (++path->tries < NGX_QUIC_PATH_RETRIES) {
> > +        pto = ngx_quic_pto(c, ctx) << path->tries;
> > +        path->expires = ngx_current_msec + pto;
> > +
> 
> Setting path->expires there seems useless.
> It don't seem to be used in ngx_quic_send_path_mtu_probe(),
> and ngx_quic_discover_path_mtu() used to reset path->expires.

If ngx_quic_send_path_mtu_probe() below returns NGX_OK, the path should stay
in the same state, but expiration time should be advanced.  To make it
more obvious, I changed this code.

> > +        rc = ngx_quic_send_path_mtu_probe(c, path);
> > +        if (rc != NGX_DECLINED) {
> > +            return rc;
> >         }
> >     }
> > 
> > -    if (next != -1) {
> > -        ngx_add_timer(&qc->path_validation, next);
> > +    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
> > +                   "quic path seq:%uL expired mtu:%uz",
> > +                   path->seqnum, path->mtud);
> > +
> > +    path->max_mtu = path->mtud;
> > +
> > +    ngx_quic_discover_path_mtu(c, path);
> > +
> > +    return NGX_OK;
> > +}
> > +
> > +
> > +static ngx_int_t
> > +ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path)
> > +{
> > +    ngx_int_t               rc;
> > +    ngx_uint_t              log_error;
> > +    ngx_quic_frame_t        frame;
> > +    ngx_quic_send_ctx_t    *ctx;
> > +    ngx_quic_connection_t  *qc;
> > +
> > +    ngx_memzero(&frame, sizeof(ngx_quic_frame_t));
> > +
> > +    frame.level = ssl_encryption_application;
> > +    frame.type = NGX_QUIC_FT_PING;
> > +
> > +    qc = ngx_quic_get_connection(c);
> > +    ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> > +    path->mtu_pnum[path->tries] = ctx->pnum;
> > +
> > +    ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
> > +                   "quic path seq:%uL send probe "
> > +                   "mtu:%uz pnum:%uL tries:%ui",
> > +                   path->seqnum, path->mtud, ctx->pnum, path->tries);
> > +
> > +    log_error = c->log_error;
> > +    c->log_error = NGX_ERROR_IGNORE_EMSGSIZE;
> > +
> > +    rc = ngx_quic_frame_sendto(c, &frame, path->mtud, path);
> > +    c->log_error = log_error;
> > +
> > +    if (rc == NGX_ERROR) {
> > +        if (c->write->error) {
> > +            c->write->error = 0;
> > +
> > +            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
> > +                           "quic path seq:%uL rejected mtu:%uz",
> > +                           path->seqnum, path->mtud);
> > +
> > +            return NGX_DECLINED;
> 
> The error handling looks awkward to decline on write error.
> ngx_sendmsg() sets write error on fatal sendmsg errors,
> and keeps it unset NGX_EAGAIN and NGX_EINTR, which
> are typically non-fatal transient errors.
> I understand this is done mostly to ignore EMSGSIZE,
> maybe it makes sense to ignore the rest sendmsg() errors.

The problem is, the error doesn't make it to this point.  All be have here is
NGX_ERROR and c->write->error.  I think, for PMTUD purposes we can safely
ignore the errors.  It's hinghly likely, the error will repeat while doing
connection I/O and we'll shortly close the connection.

> > +        }
> > +
> > +        return NGX_ERROR;
> >     }
> > +
> > +    return NGX_OK;
> > }
> > +
> > +
> > +ngx_int_t
> > +ngx_quic_handle_path_mtu_ack(ngx_connection_t *c, ngx_quic_path_t *path,
> > +    uint64_t min, uint64_t max)
> > +{
> > +    uint64_t    pnum;
> > +    ngx_uint_t  i;
> > +
> > +    if (path->state != NGX_QUIC_PATH_DISCOVERING_MTU) {
> > +        return NGX_OK;
> > +    }
> > +
> > +    for (i = 0; i < NGX_QUIC_PATH_RETRIES; i++) {
> > +        pnum = path->mtu_pnum[i];
> > +
> > +        if (pnum == NGX_QUIC_UNSET_PN) {
> > +            break;
> > +        }
> > +
> > +        if (pnum < min || pnum > max) {
> > +            continue;
> > +        }
> > +
> > +        path->mtu = path->mtud;
> > +
> > +        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
> > +                       "quic path seq:%uL ack mtu:%uz",
> > +                       path->seqnum, path->mtu);
> > +
> > +        ngx_quic_discover_path_mtu(c, path);
> > +
> > +        break;
> 
> What if two probes appeared ACKed in one range.
> Though that seems to be safe to break there, as two
> probes barely have different probe sizes, rather retries.

No matter how many probes are acked, the next step is still the same.
So there's no reason to find all of them.

> > +    }
> > +
> > +    return NGX_OK;
> > +}
> > diff --git a/src/event/quic/ngx_event_quic_migration.h b/src/event/quic/ngx_event_quic_migration.h
> > --- a/src/event/quic/ngx_event_quic_migration.h
> > +++ b/src/event/quic/ngx_event_quic_migration.h
> > @@ -18,10 +18,10 @@
> > #define NGX_QUIC_PATH_BACKUP    2
> > 
> > #define ngx_quic_path_dbg(c, msg, path)                                       \
> > -    ngx_log_debug6(NGX_LOG_DEBUG_EVENT, c->log, 0,                            \
> > -                   "quic path seq:%uL %s sent:%O recvd:%O state:%s%s",        \
> > +    ngx_log_debug7(NGX_LOG_DEBUG_EVENT, c->log, 0,                            \
> > +                   "quic path seq:%uL %s tx:%O rx:%O valid:%d st:%d mtu:%uz", \
> >                    path->seqnum, msg, path->sent, path->received,             \
> > -                   path->validated ? "V": "N", path->validating ? "R": "");
> > +                   path->validated, path->state, path->mtu);
> > 
> > ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c,
> >     ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f);
> > @@ -36,6 +36,10 @@ ngx_int_t ngx_quic_set_path(ngx_connecti
> > ngx_int_t ngx_quic_handle_migration(ngx_connection_t *c,
> >     ngx_quic_header_t *pkt);
> > 
> > -void ngx_quic_path_validation_handler(ngx_event_t *ev);
> > +void ngx_quic_path_handler(ngx_event_t *ev);
> > +
> > +void ngx_quic_discover_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path);
> > +ngx_int_t ngx_quic_handle_path_mtu_ack(ngx_connection_t *c,
> > +    ngx_quic_path_t *path, uint64_t min, uint64_t max);
> 
> Nitpicking:
> maybe just ngx_quic_handle_path_mtu() ?
> This would match ngx_quic_discover_path_mtu() naming.

OK, renamed.

> > 
> > #endif /* _NGX_EVENT_QUIC_MIGRATION_H_INCLUDED_ */
> > diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
> > --- a/src/event/quic/ngx_event_quic_output.c
> > +++ b/src/event/quic/ngx_event_quic_output.c
> > @@ -10,9 +10,6 @@
> > #include <ngx_event_quic_connection.h>
> > 
> > 
> > -#define NGX_QUIC_MAX_UDP_PAYLOAD_OUT   1252
> > -#define NGX_QUIC_MAX_UDP_PAYLOAD_OUT6  1232
> > -
> > #define NGX_QUIC_MAX_UDP_SEGMENT_BUF  65487 /* 65K - IPv6 header */
> > #define NGX_QUIC_MAX_SEGMENTS            64 /* UDP_MAX_SEGMENTS */
> > 
> > @@ -61,21 +58,6 @@ static size_t ngx_quic_path_limit(ngx_co
> >     size_t size);
> > 
> > 
> > -size_t
> > -ngx_quic_max_udp_payload(ngx_connection_t *c)
> > -{
> > -    /* TODO: path MTU discovery */
> > -
> > -#if (NGX_HAVE_INET6)
> > -    if (c->sockaddr->sa_family == AF_INET6) {
> > -        return NGX_QUIC_MAX_UDP_PAYLOAD_OUT6;
> > -    }
> > -#endif
> > -
> > -    return NGX_QUIC_MAX_UDP_PAYLOAD_OUT;
> > -}
> > -
> > -
> > ngx_int_t
> > ngx_quic_output(ngx_connection_t *c)
> > {
> > @@ -142,10 +124,7 @@ ngx_quic_create_datagrams(ngx_connection
> > 
> >         p = dst;
> > 
> > -        len = ngx_min(qc->ctp.max_udp_payload_size,
> > -                      NGX_QUIC_MAX_UDP_PAYLOAD_SIZE);
> > -
> > -        len = ngx_quic_path_limit(c, path, len);
> > +        len = ngx_quic_path_limit(c, path, path->mtu);
> > 
> >         pad = ngx_quic_get_padding_level(c);
> > 
> > @@ -299,9 +278,7 @@ ngx_quic_allow_segmentation(ngx_connecti
> >     ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
> > 
> >     bytes = 0;
> > -
> > -    len = ngx_min(qc->ctp.max_udp_payload_size,
> > -                  NGX_QUIC_MAX_UDP_SEGMENT_BUF);
> > +    len = ngx_min(qc->path->mtu, NGX_QUIC_MAX_UDP_SEGMENT_BUF);
> > 
> >     for (q = ngx_queue_head(&ctx->frames);
> >          q != ngx_queue_sentinel(&ctx->frames);
> > @@ -345,8 +322,7 @@ ngx_quic_create_segments(ngx_connection_
> >         return NGX_ERROR;
> >     }
> > 
> > -    segsize = ngx_min(qc->ctp.max_udp_payload_size,
> > -                      NGX_QUIC_MAX_UDP_SEGMENT_BUF);
> > +    segsize = ngx_min(path->mtu, NGX_QUIC_MAX_UDP_SEGMENT_BUF);
> >     p = dst;
> >     end = dst + sizeof(dst);
> > 
> > diff --git a/src/event/quic/ngx_event_quic_output.h b/src/event/quic/ngx_event_quic_output.h
> > --- a/src/event/quic/ngx_event_quic_output.h
> > +++ b/src/event/quic/ngx_event_quic_output.h
> > @@ -12,8 +12,6 @@
> > #include <ngx_core.h>
> > 
> > 
> > -size_t ngx_quic_max_udp_payload(ngx_connection_t *c);
> > -
> > ngx_int_t ngx_quic_output(ngx_connection_t *c);
> > 
> > ngx_int_t ngx_quic_negotiate_version(ngx_connection_t *c,
> > diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c
> > --- a/src/event/quic/ngx_event_quic_ssl.c
> > +++ b/src/event/quic/ngx_event_quic_ssl.c
> > @@ -494,6 +494,8 @@ ngx_quic_crypto_input(ngx_connection_t *
> >      */
> >     ngx_quic_discard_ctx(c, ssl_encryption_handshake);
> > 
> > +    ngx_quic_discover_path_mtu(c, qc->path);
> > +
> >     /* start accepting clients on negotiated number of server ids */
> >     if (ngx_quic_create_sockets(c) != NGX_OK) {
> >         return NGX_ERROR;
> > diff --git a/src/os/unix/ngx_errno.h b/src/os/unix/ngx_errno.h
> > --- a/src/os/unix/ngx_errno.h
> > +++ b/src/os/unix/ngx_errno.h
> > @@ -54,6 +54,7 @@ typedef int               ngx_err_t;
> > #define NGX_ENOMOREFILES  0
> > #define NGX_ELOOP         ELOOP
> > #define NGX_EBADF         EBADF
> > +#define NGX_EMSGSIZE      EMSGSIZE
> > 
> > #if (NGX_HAVE_OPENAT)
> > #define NGX_EMLINK        EMLINK
> > _______________________________________________
> > nginx-devel mailing list
> > nginx-devel at nginx.org
> > https://mailman.nginx.org/mailman/listinfo/nginx-devel
> 
> -- 
> Sergey Kandaurov
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx-devel

The diff is attached.

--
Roman Arutyunyan
-------------- next part --------------
# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1690824524 -14400
#      Mon Jul 31 21:28:44 2023 +0400
# Node ID a9f81e4b150735f01ac78a3ab5363f727b0f3b26
# Parent  ff84b813841c44f5ff326031732fab16a437c364
[mq]: quic-pmtud-fix

diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c
--- a/src/event/quic/ngx_event_quic_ack.c
+++ b/src/event/quic/ngx_event_quic_ack.c
@@ -230,7 +230,7 @@ ngx_quic_handle_ack_frame_range(ngx_conn
     qc = ngx_quic_get_connection(c);
 
     if (ctx->level == ssl_encryption_application) {
-        if (ngx_quic_handle_path_mtu_ack(c, qc->path, min, max) != NGX_OK) {
+        if (ngx_quic_handle_path_mtu(c, qc->path, min, max) != NGX_OK) {
             return NGX_ERROR;
         }
     }
diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h
--- a/src/event/quic/ngx_event_quic_connection.h
+++ b/src/event/quic/ngx_event_quic_connection.h
@@ -70,7 +70,7 @@ typedef enum {
     NGX_QUIC_PATH_IDLE = 0,
     NGX_QUIC_PATH_VALIDATING,
     NGX_QUIC_PATH_WAITING,
-    NGX_QUIC_PATH_DISCOVERING_MTU
+    NGX_QUIC_PATH_MTUD
 } ngx_quic_path_state_e;
 
 
diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c
--- a/src/event/quic/ngx_event_quic_migration.c
+++ b/src/event/quic/ngx_event_quic_migration.c
@@ -10,7 +10,7 @@
 #include <ngx_event_quic_connection.h>
 
 
-#define NGX_QUIC_PATH_MTU_DELAY       1000
+#define NGX_QUIC_PATH_MTU_DELAY       100
 #define NGX_QUIC_PATH_MTU_STEP        256
 #define NGX_QUIC_PATH_MTU_PRECISION   16
 
@@ -686,7 +686,7 @@ ngx_quic_path_handler(ngx_event_t *ev)
 
         switch (path->state) {
         case NGX_QUIC_PATH_VALIDATING:
-             if (ngx_quic_expire_path_validation(c, path) != NGX_OK) {
+            if (ngx_quic_expire_path_validation(c, path) != NGX_OK) {
                 goto failed;
             }
 
@@ -699,17 +699,17 @@ ngx_quic_path_handler(ngx_event_t *ev)
 
             break;
 
-        case NGX_QUIC_PATH_DISCOVERING_MTU:
+        case NGX_QUIC_PATH_MTUD:
             if (ngx_quic_expire_path_mtu_discovery(c, path) != NGX_OK) {
                 goto failed;
-             }
+            }
 
             break;
 
         default:
             break;
-         }
-     }
+        }
+    }
 
     ngx_quic_set_path_timer(c);
 
@@ -812,7 +812,7 @@ ngx_quic_expire_path_mtu_delay(ngx_conne
         if (rc == NGX_OK) {
             pto = ngx_quic_pto(c, ctx);
             path->expires = ngx_current_msec + pto;
-            path->state = NGX_QUIC_PATH_DISCOVERING_MTU;
+            path->state = NGX_QUIC_PATH_MTUD;
             return NGX_OK;
         }
 
@@ -842,13 +842,19 @@ ngx_quic_expire_path_mtu_discovery(ngx_c
     ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
 
     if (++path->tries < NGX_QUIC_PATH_RETRIES) {
-        pto = ngx_quic_pto(c, ctx) << path->tries;
-        path->expires = ngx_current_msec + pto;
+        rc = ngx_quic_send_path_mtu_probe(c, path);
+
+        if (rc == NGX_ERROR) {
+            return NGX_ERROR;
+        }
 
-        rc = ngx_quic_send_path_mtu_probe(c, path);
-        if (rc != NGX_DECLINED) {
-            return rc;
+        if (rc == NGX_OK) {
+            pto = ngx_quic_pto(c, ctx) << path->tries;
+            path->expires = ngx_current_msec + pto;
+            return NGX_OK;
         }
+
+        /* rc == NGX_DECLINED */
     }
 
     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
@@ -911,13 +917,13 @@ ngx_quic_send_path_mtu_probe(ngx_connect
 
 
 ngx_int_t
-ngx_quic_handle_path_mtu_ack(ngx_connection_t *c, ngx_quic_path_t *path,
+ngx_quic_handle_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path,
     uint64_t min, uint64_t max)
 {
     uint64_t    pnum;
     ngx_uint_t  i;
 
-    if (path->state != NGX_QUIC_PATH_DISCOVERING_MTU) {
+    if (path->state != NGX_QUIC_PATH_MTUD) {
         return NGX_OK;
     }
 
diff --git a/src/event/quic/ngx_event_quic_migration.h b/src/event/quic/ngx_event_quic_migration.h
--- a/src/event/quic/ngx_event_quic_migration.h
+++ b/src/event/quic/ngx_event_quic_migration.h
@@ -39,7 +39,7 @@ ngx_int_t ngx_quic_handle_migration(ngx_
 void ngx_quic_path_handler(ngx_event_t *ev);
 
 void ngx_quic_discover_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path);
-ngx_int_t ngx_quic_handle_path_mtu_ack(ngx_connection_t *c,
+ngx_int_t ngx_quic_handle_path_mtu(ngx_connection_t *c,
     ngx_quic_path_t *path, uint64_t min, uint64_t max);
 
 #endif /* _NGX_EVENT_QUIC_MIGRATION_H_INCLUDED_ */