+ rev="90">

@@ -88,11 +88,6 @@ versions: -22.10 “kinetic” -x86_64, aarch64/arm64 - - - 23.04 “lunar” x86_64, aarch64/arm64 diff -r ac4191d05fdf -r 1ad61bfc7630 xml/ru/linux_packages.xml --- a/xml/ru/linux_packages.xml Thu Sep 14 21:20:14 2023 +0100 +++ b/xml/ru/linux_packages.xml Fri Sep 22 15:11:23 2023 -0700 @@ -7,7 +7,7 @@

+ rev="90">

@@ -88,11 +88,6 @@ -22.10 “kinetic” -x86_64, aarch64/arm64 - - - 23.04 “lunar” x86_64, aarch64/arm64 From mdounin at mdounin.ru Fri Sep 22 22:17:00 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sat, 23 Sep 2023 01:17:00 +0300 Subject: [PATCH] Linux packages: removed Ubuntu 22.10 'kinetic' due to EOL In-Reply-To: <1ad61bfc7630adf1d646.1695420743@qgcd7xg9r9.olympus.f5net.com> References: <1ad61bfc7630adf1d646.1695420743@qgcd7xg9r9.olympus.f5net.com> Message-ID: Hello! On Fri, Sep 22, 2023 at 03:12:23PM -0700, Konstantin Pavlov wrote: > # HG changeset patch > # User Konstantin Pavlov > # Date 1695420683 25200 > # Fri Sep 22 15:11:23 2023 -0700 > # Node ID 1ad61bfc7630adf1d6460cf84cec484de4017326 > # Parent ac4191d05fdf12dbc977a3a26dfde2799d301283 > Linux packages: removed Ubuntu 22.10 'kinetic' due to EOL. > > diff -r ac4191d05fdf -r 1ad61bfc7630 xml/en/linux_packages.xml > --- a/xml/en/linux_packages.xml Thu Sep 14 21:20:14 2023 +0100 > +++ b/xml/en/linux_packages.xml Fri Sep 22 15:11:23 2023 -0700 > @@ -7,7 +7,7 @@ >

link="/en/linux_packages.html" > lang="en" > - rev="89"> > + rev="90"> > >

> > @@ -88,11 +88,6 @@ versions: > > > > -22.10 “kinetic” > -x86_64, aarch64/arm64 > - > - > - > 23.04 “lunar” > x86_64, aarch64/arm64 > > diff -r ac4191d05fdf -r 1ad61bfc7630 xml/ru/linux_packages.xml > --- a/xml/ru/linux_packages.xml Thu Sep 14 21:20:14 2023 +0100 > +++ b/xml/ru/linux_packages.xml Fri Sep 22 15:11:23 2023 -0700 > @@ -7,7 +7,7 @@ >

link="/ru/linux_packages.html" > lang="ru" > - rev="89"> > + rev="90"> > >

> > @@ -88,11 +88,6 @@ > > > > -22.10 “kinetic” > -x86_64, aarch64/arm64 > - > - > - > 23.04 “lunar” > x86_64, aarch64/arm64 > Looks fine. -- Maxim Dounin http://mdounin.ru/ From xeioex at nginx.com Sat Sep 23 00:39:18 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Sat, 23 Sep 2023 00:39:18 +0000 Subject: [njs] Modules: added debug log for vm cloning and destroying. Message-ID: details: https://hg.nginx.org/njs/rev/dbb011e433b2 branches: changeset: 2206:dbb011e433b2 user: Dmitry Volyntsev date: Fri Sep 22 13:00:04 2023 -0700 description: Modules: added debug log for vm cloning and destroying. diffstat: nginx/ngx_http_js_module.c | 6 ++++++ nginx/ngx_stream_js_module.c | 6 ++++++ 2 files changed, 12 insertions(+), 0 deletions(-) diffs (46 lines): diff -r fc98c27d3fc0 -r dbb011e433b2 nginx/ngx_http_js_module.c --- a/nginx/ngx_http_js_module.c Thu Sep 21 12:40:53 2023 -0700 +++ b/nginx/ngx_http_js_module.c Fri Sep 22 13:00:04 2023 -0700 @@ -1366,6 +1366,9 @@ ngx_http_js_init_vm(ngx_http_request_t * return NGX_ERROR; } + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http js vm clone: %p from: %p", ctx->vm, jlcf->vm); + cln = ngx_pool_cleanup_add(r->pool, 0); if (cln == NULL) { return NGX_ERROR; @@ -1425,6 +1428,9 @@ ngx_http_js_cleanup_ctx(void *data) ngx_log_error(NGX_LOG_ERR, ctx->log, 0, "pending events"); } + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->log, 0, "http js vm destroy: %p", + ctx->vm); + njs_vm_destroy(ctx->vm); } diff -r fc98c27d3fc0 -r dbb011e433b2 nginx/ngx_stream_js_module.c --- a/nginx/ngx_stream_js_module.c Thu Sep 21 12:40:53 2023 -0700 +++ b/nginx/ngx_stream_js_module.c Fri Sep 22 13:00:04 2023 -0700 @@ -1030,6 +1030,9 @@ ngx_stream_js_init_vm(ngx_stream_session return NGX_ERROR; } + ngx_log_debug2(NGX_LOG_DEBUG_STREAM, s->connection->log, 0, + "stream js vm clone: %p from: %p", ctx->vm, jscf->vm); + cln = ngx_pool_cleanup_add(s->connection->pool, 0); if (cln == NULL) { return NGX_ERROR; @@ -1107,6 +1110,9 @@ ngx_stream_js_cleanup(void *data) ngx_log_error(NGX_LOG_ERR, s->connection->log, 0, "pending events"); } + ngx_log_debug1(NGX_LOG_DEBUG_STREAM, s->connection->log, 0, + "stream js vm destroy: %p", ctx->vm); + njs_vm_destroy(ctx->vm); } From xeioex at nginx.com Sat Sep 23 00:39:20 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Sat, 23 Sep 2023 00:39:20 +0000 Subject: [njs] Introduced API to work with external value pointers. Message-ID: details: https://hg.nginx.org/njs/rev/cf7e8f006bd8 branches: changeset: 2207:cf7e8f006bd8 user: Dmitry Volyntsev date: Fri Sep 22 13:00:05 2023 -0700 description: Introduced API to work with external value pointers. This allows to allocate the necessary context for external code on demand. diffstat: src/njs.h | 5 + src/njs_value.c | 25 ++++++++ src/test/njs_externals_test.c | 117 ++++++++++++++++++++++++++++++++++++++++++ src/test/njs_unit_test.c | 16 +++++- 4 files changed, 162 insertions(+), 1 deletions(-) diffs (265 lines): diff -r dbb011e433b2 -r cf7e8f006bd8 src/njs.h --- a/src/njs.h Fri Sep 22 13:00:04 2023 -0700 +++ b/src/njs.h Fri Sep 22 13:00:05 2023 -0700 @@ -532,12 +532,15 @@ NJS_EXPORT void njs_value_boolean_set(nj NJS_EXPORT void njs_value_number_set(njs_value_t *value, double num); NJS_EXPORT void njs_value_function_set(njs_value_t *value, njs_function_t *function); +NJS_EXPORT void njs_value_external_set(njs_value_t *value, + njs_external_ptr_t external); NJS_EXPORT uint8_t njs_value_bool(const njs_value_t *value); NJS_EXPORT double njs_value_number(const njs_value_t *value); NJS_EXPORT njs_function_t *njs_value_function(const njs_value_t *value); NJS_EXPORT njs_function_native_t njs_value_native_function( const njs_value_t *value); +njs_external_ptr_t njs_value_external(const njs_value_t *value); NJS_EXPORT njs_int_t njs_value_external_tag(const njs_value_t *value); NJS_EXPORT uint16_t njs_vm_prop_magic16(njs_object_prop_t *prop); @@ -555,6 +558,8 @@ NJS_EXPORT njs_int_t njs_value_is_valid_ NJS_EXPORT njs_int_t njs_value_is_string(const njs_value_t *value); NJS_EXPORT njs_int_t njs_value_is_object(const njs_value_t *value); NJS_EXPORT njs_int_t njs_value_is_error(const njs_value_t *value); +NJS_EXPORT njs_int_t njs_value_is_external(const njs_value_t *value, + njs_int_t proto_id); NJS_EXPORT njs_int_t njs_value_is_array(const njs_value_t *value); NJS_EXPORT njs_int_t njs_value_is_function(const njs_value_t *value); NJS_EXPORT njs_int_t njs_value_is_buffer(const njs_value_t *value); diff -r dbb011e433b2 -r cf7e8f006bd8 src/njs_value.c --- a/src/njs_value.c Fri Sep 22 13:00:04 2023 -0700 +++ b/src/njs_value.c Fri Sep 22 13:00:05 2023 -0700 @@ -466,6 +466,15 @@ njs_value_function_set(njs_value_t *valu } +void +njs_value_external_set(njs_value_t *value, njs_external_ptr_t external) +{ + njs_assert(njs_value_is_external(value, NJS_PROTO_ID_ANY)); + + njs_object_data(value) = external; +} + + uint8_t njs_value_bool(const njs_value_t *value) { @@ -504,6 +513,15 @@ njs_value_native_function(const njs_valu } +njs_external_ptr_t +njs_value_external(const njs_value_t *value) +{ + njs_assert(njs_value_is_external(value, NJS_PROTO_ID_ANY)); + + return njs_object_data(value); +} + + njs_int_t njs_value_is_null(const njs_value_t *value) { @@ -577,6 +595,13 @@ njs_value_is_error(const njs_value_t *va njs_int_t +njs_value_is_external(const njs_value_t *value, njs_int_t proto_id) +{ + return njs_is_object_data(value, njs_make_tag(proto_id)); +} + + +njs_int_t njs_value_is_array(const njs_value_t *value) { return njs_is_array(value); diff -r dbb011e433b2 -r cf7e8f006bd8 src/test/njs_externals_test.c --- a/src/test/njs_externals_test.c Fri Sep 22 13:00:04 2023 -0700 +++ b/src/test/njs_externals_test.c Fri Sep 22 13:00:05 2023 -0700 @@ -33,6 +33,7 @@ njs_int_t njs_array_buffer_detach(njs_vm static njs_int_t njs_external_r_proto_id; +static njs_int_t njs_external_null_proto_id; static njs_int_t njs_external_error_ctor_id; @@ -556,6 +557,67 @@ njs_unit_test_r_bind(njs_vm_t *vm, njs_v static njs_int_t +njs_unit_test_null_get(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t unused, njs_value_t *retval) +{ + double *d; + njs_value_t *this; + + this = njs_argument(args, 0); + + if (!njs_value_is_external(this, njs_external_null_proto_id)) { + njs_type_error(vm, "\"this\" is not a null external"); + return NJS_ERROR; + } + + d = njs_value_external(this); + + if (d == NULL) { + njs_value_undefined_set(retval); + + } else { + njs_value_number_set(retval, *d); + } + + return NJS_OK; +} + + +static njs_int_t +njs_unit_test_null_set(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t unused, njs_value_t *retval) +{ + double *d; + njs_value_t *this; + + this = njs_argument(args, 0); + + if (!njs_value_is_external(this, njs_external_null_proto_id)) { + njs_type_error(vm, "\"this\" is not a null external"); + return NJS_ERROR; + } + + d = njs_value_external(this); + + if (d == NULL) { + d = njs_mp_alloc(vm->mem_pool, sizeof(double)); + if (d == NULL) { + njs_memory_error(vm); + return NJS_ERROR; + } + } + + *d = njs_value_number(njs_arg(args, nargs, 1)); + + njs_value_external_set(this, d); + + njs_value_undefined_set(retval); + + return NJS_OK; +} + + +static njs_int_t njs_unit_test_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused, njs_value_t *retval) { @@ -1043,6 +1105,40 @@ static njs_external_t njs_unit_test_r_e }; +static njs_external_t njs_unit_test_null_external[] = { + + { + .flags = NJS_EXTERN_PROPERTY | NJS_EXTERN_SYMBOL, + .name.symbol = NJS_SYMBOL_TO_STRING_TAG, + .u.property = { + .value = "Null", + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("get"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = njs_unit_test_null_get, + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("set"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = njs_unit_test_null_set, + } + }, + +}; + static njs_external_t njs_unit_test_ctor_props[] = { @@ -1170,6 +1266,7 @@ njs_externals_init_internal(njs_vm_t *vm njs_unit_test_prop_t *prop; static const njs_str_t external_ctor = njs_str("ExternalConstructor"); + static const njs_str_t external_null = njs_str("ExternalNull"); static const njs_str_t external_error = njs_str("ExternalError"); if (shared) { @@ -1195,6 +1292,26 @@ njs_externals_init_internal(njs_vm_t *vm return NJS_ERROR; } + njs_external_null_proto_id = njs_vm_external_prototype(vm, + njs_unit_test_null_external, + njs_nitems(njs_unit_test_null_external)); + if (njs_slow_path(njs_external_null_proto_id < 0)) { + njs_printf("njs_vm_external_prototype() failed\n"); + return NJS_ERROR; + } + + ret = njs_vm_external_create(vm, njs_value_arg(&value), + njs_external_null_proto_id, NULL, 1); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + ret = njs_vm_bind(vm, &external_null, njs_value_arg(&value), 1); + if (njs_slow_path(ret != NJS_OK)) { + njs_printf("njs_vm_bind() failed\n"); + return NJS_ERROR; + } + njs_external_error_ctor_id = njs_vm_external_constructor(vm, &external_error, njs_error_constructor, njs_unit_test_ctor_props, diff -r dbb011e433b2 -r cf7e8f006bd8 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Fri Sep 22 13:00:04 2023 -0700 +++ b/src/test/njs_unit_test.c Fri Sep 22 13:00:05 2023 -0700 @@ -22637,7 +22637,8 @@ static njs_unit_test_t njs_externals_te #endif { njs_str("Object.keys(this).sort()"), - njs_str("$262,$r,$r2,$r3,$shared,ExternalConstructor,ExternalError," NCRYPTO "global,njs,process") }, + njs_str("$262,$r,$r2,$r3,$shared,ExternalConstructor,ExternalError," + "ExternalNull," NCRYPTO "global,njs,process") }, { njs_str("Object.getOwnPropertySymbols($r2)[0] == Symbol.toStringTag"), njs_str("true") }, @@ -22924,6 +22925,19 @@ static njs_unit_test_t njs_shared_test[ { njs_str("var sum = new Function('a, b', 'return a + b');" "sum(2, 4);"), njs_str("6") }, + + { njs_str("ExternalNull.get()"), + njs_str("undefined") }, + + { njs_str("ExternalNull.set(37); ExternalNull.get()"), + njs_str("37") }, + + { njs_str("ExternalNull.set(23); ExternalNull.set(37); ExternalNull.get()"), + njs_str("37") }, + + { njs_str("var v = Math.round(Math.random() * 1000); ExternalNull.set(v);" + "ExternalNull.get() == v"), + njs_str("true") }, }; From xeioex at nginx.com Sat Sep 23 00:39:22 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Sat, 23 Sep 2023 00:39:22 +0000 Subject: [njs] Modules: introduced console object. Message-ID: details: https://hg.nginx.org/njs/rev/ed935fa4805b branches: changeset: 2208:ed935fa4805b user: Dmitry Volyntsev date: Fri Sep 22 13:00:06 2023 -0700 description: Modules: introduced console object. The following methods were added: dump(), error(), info(), log(), time(), timeEnd(), warn(). diffstat: nginx/ngx_js.c | 365 ++++++++++++++++++++++++++++++++++++++++++++++++-- nginx/t/js_console.t | 149 ++++++++++++++++++++ 2 files changed, 499 insertions(+), 15 deletions(-) diffs (592 lines): diff -r cf7e8f006bd8 -r ed935fa4805b nginx/ngx_js.c --- a/nginx/ngx_js.c Fri Sep 22 13:00:05 2023 -0700 +++ b/nginx/ngx_js.c Fri Sep 22 13:00:06 2023 -0700 @@ -11,6 +11,18 @@ #include "ngx_js.h" +typedef struct { + ngx_queue_t labels; +} ngx_js_console_t; + + +typedef struct { + njs_str_t name; + uint64_t time; + ngx_queue_t queue; +} ngx_js_timelabel_t; + + static njs_int_t ngx_js_ext_build(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval, njs_value_t *retval); static njs_int_t ngx_js_ext_conf_file_path(njs_vm_t *vm, @@ -27,9 +39,14 @@ static njs_int_t ngx_js_ext_version(njs_ njs_value_t *value, njs_value_t *setval, njs_value_t *retval); static njs_int_t ngx_js_ext_worker_id(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval, njs_value_t *retval); +static njs_int_t ngx_js_ext_console_time(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t unused, njs_value_t *retval); +static njs_int_t ngx_js_ext_console_time_end(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t unused, njs_value_t *retval); static void ngx_js_cleanup_vm(void *data); static njs_int_t ngx_js_core_init(njs_vm_t *vm); +static uint64_t ngx_js_monotonic_time(void); static njs_external_t ngx_js_ext_global_shared[] = { @@ -197,6 +214,103 @@ static njs_external_t ngx_js_ext_core[] }; +static njs_external_t ngx_js_ext_console[] = { + + { + .flags = NJS_EXTERN_PROPERTY | NJS_EXTERN_SYMBOL, + .name.symbol = NJS_SYMBOL_TO_STRING_TAG, + .u.property = { + .value = "Console", + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("dump"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = ngx_js_ext_log, +#define NGX_JS_LOG_DUMP 16 +#define NGX_JS_LOG_MASK 15 + .magic8 = NGX_LOG_INFO | NGX_JS_LOG_DUMP, + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("error"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = ngx_js_ext_log, + .magic8 = NGX_LOG_ERR, + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("info"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = ngx_js_ext_log, + .magic8 = NGX_LOG_INFO, + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("log"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = ngx_js_ext_log, + .magic8 = NGX_LOG_INFO, + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("time"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = ngx_js_ext_console_time, + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("timeEnd"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = ngx_js_ext_console_time_end, + } + }, + + { + .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("warn"), + .writable = 1, + .configurable = 1, + .enumerable = 1, + .u.method = { + .native = ngx_js_ext_log, + .magic8 = NGX_LOG_WARN, + } + }, + +}; + + njs_module_t ngx_js_ngx_module = { .name = njs_str("ngx"), .preinit = NULL, @@ -210,6 +324,9 @@ njs_module_t *njs_js_addon_modules_share }; +static njs_int_t ngx_js_console_proto_id; + + ngx_int_t ngx_js_call(njs_vm_t *vm, ngx_str_t *fname, ngx_log_t *log, njs_opaque_value_t *args, njs_uint_t nargs) @@ -345,6 +462,26 @@ ngx_js_core_init(njs_vm_t *vm) return NJS_ERROR; } + ngx_js_console_proto_id = njs_vm_external_prototype(vm, ngx_js_ext_console, + njs_nitems(ngx_js_ext_console)); + if (ngx_js_console_proto_id < 0) { + return NJS_ERROR; + } + + ret = njs_vm_external_create(vm, njs_value_arg(&value), + ngx_js_console_proto_id, NULL, 1); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + name.length = 7; + name.start = (u_char *) "console"; + + ret = njs_vm_bind(vm, &name, njs_value_arg(&value), 1); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + return NJS_OK; } @@ -509,14 +646,13 @@ ngx_js_ext_worker_id(njs_vm_t *vm, njs_o njs_int_t ngx_js_ext_log(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, - njs_index_t level, njs_value_t *retval) + njs_index_t magic, njs_value_t *retval) { - char *p; - ngx_int_t lvl; - njs_str_t msg; - njs_value_t *value; - ngx_connection_t *c; - ngx_log_handler_pt handler; + char *p; + ngx_int_t lvl; + njs_str_t msg; + njs_uint_t n; + njs_log_level_t level; p = njs_vm_external(vm, NJS_PROTO_ID_ANY, njs_argument(args, 0)); if (p == NULL) { @@ -524,7 +660,7 @@ ngx_js_ext_log(njs_vm_t *vm, njs_value_t return NJS_ERROR; } - value = njs_arg(args, nargs, (level != 0) ? 1 : 2); + level = (njs_log_level_t) magic & NGX_JS_LOG_MASK; if (level == 0) { if (ngx_js_integer(vm, njs_arg(args, nargs, 1), &lvl) != NGX_OK) { @@ -532,20 +668,196 @@ ngx_js_ext_log(njs_vm_t *vm, njs_value_t } level = lvl; + n = 2; + + } else { + n = 1; + } + + for (; n < nargs; n++) { + if (njs_vm_value_dump(vm, &msg, njs_argument(args, n), 1, + !!(magic & NGX_JS_LOG_DUMP)) + == NJS_ERROR) + { + return NJS_ERROR; + } + + ngx_js_logger(vm, p, level, msg.start, msg.length); } - if (ngx_js_string(vm, value, &msg) != NGX_OK) { + njs_value_undefined_set(retval); + + return NJS_OK; +} + + +static njs_int_t +ngx_js_ext_console_time(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, + njs_index_t unused, njs_value_t *retval) +{ + njs_int_t ret; + njs_str_t name; + ngx_queue_t *labels, *q; + njs_value_t *value, *this; + ngx_js_console_t *console; + ngx_js_timelabel_t *label; + + static const njs_str_t default_label = njs_str("default"); + + this = njs_argument(args, 0); + + if (njs_slow_path(!njs_value_is_external(this, ngx_js_console_proto_id))) { + njs_vm_type_error(vm, "\"this\" is not a console external"); return NJS_ERROR; } - c = ngx_external_connection(vm, p); - handler = c->log->handler; - c->log->handler = NULL; + name = default_label; + + value = njs_arg(args, nargs, 1); + + if (njs_slow_path(!njs_value_is_string(value))) { + if (!njs_value_is_undefined(value)) { + ret = njs_value_to_string(vm, value, value); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + njs_value_string_get(value, &name); + } + + } else { + njs_value_string_get(value, &name); + } + + console = njs_value_external(this); + + if (console == NULL) { + console = njs_mp_alloc(njs_vm_memory_pool(vm), + sizeof(ngx_js_console_t)); + if (console == NULL) { + njs_vm_memory_error(vm); + return NJS_ERROR; + } + + ngx_queue_init(&console->labels); + + njs_value_external_set(this, console); + } + + labels = &console->labels; + + for (q = ngx_queue_head(labels); + q != ngx_queue_sentinel(labels); + q = ngx_queue_next(q)) + { + label = ngx_queue_data(q, ngx_js_timelabel_t, queue); + + if (njs_strstr_eq(&name, &label->name)) { + njs_vm_log(vm, "Timer \"%V\" already exists.\n", &name); + njs_value_undefined_set(retval); + return NJS_OK; + } + } + + label = njs_mp_alloc(njs_vm_memory_pool(vm), + sizeof(ngx_js_timelabel_t) + name.length); + if (njs_slow_path(label == NULL)) { + njs_vm_memory_error(vm); + return NJS_ERROR; + } + + label->name.length = name.length; + label->name.start = (u_char *) label + sizeof(ngx_js_timelabel_t); + memcpy(label->name.start, name.start, name.length); + + label->time = ngx_js_monotonic_time(); + + ngx_queue_insert_tail(&console->labels, &label->queue); + + njs_value_undefined_set(retval); + + return NJS_OK; +} + - ngx_log_error((ngx_uint_t) level, c->log, 0, "js: %*s", - msg.length, msg.start); +static njs_int_t +ngx_js_ext_console_time_end(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, + njs_index_t unused, njs_value_t *retval) +{ + uint64_t ns, ms; + njs_int_t ret; + njs_str_t name; + ngx_queue_t *labels, *q; + njs_value_t *value, *this; + ngx_js_console_t *console; + ngx_js_timelabel_t *label; + + static const njs_str_t default_label = njs_str("default"); + + ns = ngx_js_monotonic_time(); + + this = njs_argument(args, 0); + + if (njs_slow_path(!njs_value_is_external(this, ngx_js_console_proto_id))) { + njs_vm_type_error(vm, "\"this\" is not a console external"); + return NJS_ERROR; + } + + name = default_label; + + value = njs_arg(args, nargs, 1); + + if (njs_slow_path(!njs_value_is_string(value))) { + if (!njs_value_is_undefined(value)) { + ret = njs_value_to_string(vm, value, value); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + njs_value_string_get(value, &name); + } - c->log->handler = handler; + } else { + njs_value_string_get(value, &name); + } + + console = njs_value_external(this); + if (njs_slow_path(console == NULL)) { + goto not_found; + } + + labels = &console->labels; + q = ngx_queue_head(labels); + + for ( ;; ) { + if (q == ngx_queue_sentinel(labels)) { + goto not_found; + } + + label = ngx_queue_data(q, ngx_js_timelabel_t, queue); + + if (njs_strstr_eq(&name, &label->name)) { + ngx_queue_remove(&label->queue); + break; + } + + q = ngx_queue_next(q); + } + + ns = ns - label->time; + + ms = ns / 1000000; + ns = ns % 1000000; + + njs_vm_log(vm, "%V: %uL.%06uLms\n", &name, ms, ns); + + njs_value_undefined_set(retval); + + return NJS_OK; + +not_found: + + njs_vm_log(vm, "Timer \"%V\" doesn't exist.\n", &name); njs_value_undefined_set(retval); @@ -1308,3 +1620,26 @@ ngx_js_merge_conf(ngx_conf_t *cf, void * return NGX_CONF_OK; #endif } + + +static uint64_t +ngx_js_monotonic_time(void) +{ +#if (NGX_HAVE_CLOCK_MONOTONIC) + struct timespec ts; + +#if defined(CLOCK_MONOTONIC_FAST) + clock_gettime(CLOCK_MONOTONIC_FAST, &ts); +#else + clock_gettime(CLOCK_MONOTONIC, &ts); +#endif + + return (uint64_t) ts.tv_sec * 1000000000 + ts.tv_nsec; +#else + struct timeval tv; + + gettimeofday(&tv, NULL); + + return (uint64_t) tv.tv_sec * 1000000000 + tv.tv_usec * 1000; +#endif +} diff -r cf7e8f006bd8 -r ed935fa4805b nginx/t/js_console.t --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/nginx/t/js_console.t Fri Sep 22 13:00:06 2023 -0700 @@ -0,0 +1,149 @@ +#!/usr/bin/perl + +# (C) Dmitry Volyntsev +# (C) Nginx, Inc. + +# Tests for http njs module, console object. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + js_import test.js; + + server { + listen 127.0.0.1:8080; + server_name localhost; + + location /dump { + js_content test.dump; + } + + location /error { + js_content test.error; + } + + location /info { + js_content test.info; + } + + location /log { + js_content test.log; + } + + location /time { + js_content test.time; + } + + location /time_test { + js_content test.time_test; + } + + location /warn { + js_content test.warn; + } + } +} + +EOF + +$t->write_file('test.js', <try_run('no njs console')->plan(7); + +############################################################################### + +http_get('/dump?data=eyJhIjpbMiwzXX0'); +http_get('/error?data=IldBS0Ei'); +http_get('/info?data=IkJBUiI'); +http_get('/log?data=eyJhIjpbIkIiLCJDIl19'); +http_get('/time?delay=7&timer=foo'); +http_get('/time_test'); +http_get('/warn?data=IkZPTyI'); + +$t->stop(); + +like($t->read_file('error.log'), qr/\[error\].*js: WAKA/, 'console.error'); +like($t->read_file('error.log'), qr/\[info\].*js: BAR/, 'console.info'); +like($t->read_file('error.log'), qr/\[info\].*js: \{a:\['B','C'\]\}/, + 'console.log with object'); +like($t->read_file('error.log'), qr/\[warn\].*js: FOO/, 'console.warn'); +like($t->read_file('error.log'), qr/\[info\].*js: foo: \d+\.\d\d\d\d\d\dms/, + 'console.time foo'); +like($t->read_file('error.log'), qr/\[info\].*js: Timer \"default\" already/, + 'console.time already started'); +like($t->read_file('error.log'), qr/\[info\].*js: Timer \"test\" doesn't/, + 'console.time not started'); + +############################################################################### From amdeich at gmail.com Sun Sep 24 11:30:14 2023 From: amdeich at gmail.com (Andrey Kulikov) Date: Sun, 24 Sep 2023 14:30:14 +0300 Subject: [Proposal] Expose HTTP and Stream proxy modules structures for developers. Message-ID: Hello, Sometimes custom module developers need to iterate over configured location(s), configured in some or all server(s) blocks. For example, to perform post-initialization steps, defined by custom directives in their modules. Now it is virtually impossible. I've posted question about this matter few years ago: https://mailman.nginx.org/pipermail/nginx-devel/2016-October/008882.html Since then, I received quite a few private emails about whether I succeeded in my findings. So, I'm not alone, who faced this issue. Fortunately, the described goal could be easily achieved by making just internal proxy modules structures available in a separate header. After that implementation of desired functionality is just a matter of writing a few lines of code. Patches а following. No functional changes, just code movement from .c to .h files. Development community, at least through my voice, will be grateful to see them applied to the latest development branch. -- Cheers, Andrey -------------- next part -------------- An HTML attachment was scrubbed... URL: From amdeich at gmail.com Sun Sep 24 11:32:00 2023 From: amdeich at gmail.com (Andrey Kulikov) Date: Sun, 24 Sep 2023 14:32:00 +0300 Subject: [PATCH] Expose HTTP proxy module structures for module developers. Message-ID: No functional changes. Made agains latest development branch. Could be applied with patch -p1 < 0001-Expose-HTTP-proxy-module-structures-for-module-devel.patch -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: 0001-Expose-HTTP-proxy-module-structures-for-module-devel.patch Type: application/octet-stream Size: 10367 bytes Desc: not available URL: From amdeich at gmail.com Sun Sep 24 11:32:48 2023 From: amdeich at gmail.com (Andrey Kulikov) Date: Sun, 24 Sep 2023 14:32:48 +0300 Subject: [PATCH] Expose Stream proxy module structures for module developers. Message-ID: No functional changes. Made agains latest development branch. Could be applied with patch -p1 < 0001-Expose-Stream-proxy-module-structures-for-module-dev.patc -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: 0001-Expose-Stream-proxy-module-structures-for-module-dev.patch Type: application/octet-stream Size: 5991 bytes Desc: not available URL: From zaihan at unrealasia.net Sun Sep 24 16:01:11 2023 From: zaihan at unrealasia.net (Muhammad Nuzaihan) Date: Mon, 25 Sep 2023 00:01:11 +0800 Subject: NGINX status code based on *response* body Message-ID: Hi, I would like to return an error message based on the *response* body. I know i can do it easily with *request* body (with "return NGX_HTTP_BAD_REQUEST") but my case i want to do it with *response* body. Also i have tried this code with response body filter but i got "*1 header already sent, client: 127.0.0.1, server: localhost": Thank you for your guidance. Code: ``` return ngx_http_filter_finalize_request(request, &ngx_m_module, NGX_HTTP_BAD_REQUEST); ``` RESPONSE BODY CODE: ngx_int_t ResponseBodyFilter(ngx_http_request_t *request, ngx_chain_t *chain_head) { // Set the logging level to debug // TODO: remove request->connection->log->log_level = NGX_LOG_DEBUG; // Get our context so we can store the response body data FilterContext *ctx = GetModuleFilterContext(request); if (ctx == NULL) { return NGX_ERROR; } return ngx_http_filter_finalize_request(request, &ngx_my_module, NGX_HTTP_BAD_REQUEST); } ngx_int_t ModuleInit(ngx_conf_t *cf) { kNextRequestBodyFilter = ngx_http_top_request_body_filter; ngx_http_top_request_body_filter = RequestBodyFilter; kNextResponseBodyFilter = ngx_http_top_body_filter; ngx_http_top_body_filter = ResponseBodyFilter; kNextHeaderFilter = ngx_http_top_header_filter; ngx_http_top_header_filter = HeaderFilter; return NGX_OK; } From arut at nginx.com Mon Sep 25 11:20:17 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 25 Sep 2023 11:20:17 +0000 Subject: [nginx] QUIC: handle callback errors in compat. Message-ID: details: https://hg.nginx.org/nginx/rev/3db945fda515 branches: changeset: 9164:3db945fda515 user: Vladimir Khomutov date: Fri Sep 22 19:23:57 2023 +0400 description: QUIC: handle callback errors in compat. The error may be triggered in add_handhshake_data() by incorrect transport parameter sent by client. The expected behaviour in this case is to close connection complaining about incorrect parameter. Currently the connection just times out. diffstat: src/event/quic/ngx_event_quic_openssl_compat.c | 14 ++++++++++++-- 1 files changed, 12 insertions(+), 2 deletions(-) diffs (35 lines): diff -r 32b5aaebcca5 -r 3db945fda515 src/event/quic/ngx_event_quic_openssl_compat.c --- a/src/event/quic/ngx_event_quic_openssl_compat.c Wed Sep 13 17:48:15 2023 +0400 +++ b/src/event/quic/ngx_event_quic_openssl_compat.c Fri Sep 22 19:23:57 2023 +0400 @@ -408,7 +408,9 @@ ngx_quic_compat_message_callback(int wri "quic compat tx %s len:%uz ", ngx_quic_level_name(level), len); - (void) com->method->add_handshake_data(ssl, level, buf, len); + if (com->method->add_handshake_data(ssl, level, buf, len) != 1) { + goto failed; + } break; @@ -420,11 +422,19 @@ ngx_quic_compat_message_callback(int wri "quic compat %s alert:%ui len:%uz ", ngx_quic_level_name(level), alert, len); - (void) com->method->send_alert(ssl, level, alert); + if (com->method->send_alert(ssl, level, alert) != 1) { + goto failed; + } } break; } + + return; + +failed: + + ngx_post_event(&qc->close, &ngx_posted_events); } From arut at nginx.com Mon Sep 25 11:21:55 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 25 Sep 2023 15:21:55 +0400 Subject: [PATCH] QUIC openssl compat mode error handling In-Reply-To: References: <20230922152711.amodnb2asarxhmev@N00W24XTQX> Message-ID: Hi, > On 22 Sep 2023, at 19:58, Vladimir Homutov wrote: > > On Fri, Sep 22, 2023 at 07:30:50PM +0400, Roman Arutyunyan wrote: >> Hi Vladimir, >> >> On Fri, Sep 22, 2023 at 03:44:08PM +0300, Vladimir Homutov via nginx-devel wrote: >>> # HG changeset patch >>> # User Vladimir Khomutov >>> # Date 1695386443 -10800 >>> # Fri Sep 22 15:40:43 2023 +0300 >>> # Node ID 974ba23e68909ba708616410aa77074213d4d1e5 >>> # Parent 5741eddf82e826766cd0f5ec7c6fe383145ca581 >>> QUIC: handle add_handhshake_data() callback errors in compat. >>> >>> The error may be triggered by incorrect transport parameter sent by client. >>> The expected behaviour in this case is to close connection complaining >>> about incorrect parameter. Currently the connection just times out. >>> >>> diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c >>> --- a/src/event/quic/ngx_event_quic_openssl_compat.c >>> +++ b/src/event/quic/ngx_event_quic_openssl_compat.c >>> @@ -408,7 +408,10 @@ ngx_quic_compat_message_callback(int wri >>> "quic compat tx %s len:%uz ", >>> ngx_quic_level_name(level), len); >>> >>> - (void) com->method->add_handshake_data(ssl, level, buf, len); >>> + if (com->method->add_handshake_data(ssl, level, buf, len) != 1) { >>> + ngx_post_event(&qc->close, &ngx_posted_events); >>> + return; >>> + } >>> >>> break; >> >> Thanks for the patch. Indeed, it's a simple way to handle errors in callbacks. >> I'd also handle the error in send_alert(), even though we don't generate any >> errors in it now. > > Yes, although I was not sure if we need to close connection if we failed > to send alert (but probably if we are sending it, everything is already > bad enough). In either case, handling both cases similarly looks > as a way to go. I assume, send_alert() would return an error on a fatal condition, not just a local error. > >> >> -- >> Roman Arutyunyan > >> # HG changeset patch >> # User Vladimir Khomutov > >> # Date 1695396237 -14400 >> # Fri Sep 22 19:23:57 2023 +0400 >> # Node ID 3db945fda515014d220151046d02f3960bcfca0a >> # Parent 32b5aaebcca51854de6e1f8a40798edb13662edb >> QUIC: handle callback errors in compat. >> >> The error may be triggered in add_handhshake_data() by incorrect transport >> parameter sent by client. The expected behaviour in this case is to close >> connection complaining about incorrect parameter. Currently the connection >> just times out. >> >> diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c >> --- a/src/event/quic/ngx_event_quic_openssl_compat.c >> +++ b/src/event/quic/ngx_event_quic_openssl_compat.c >> @@ -408,7 +408,9 @@ ngx_quic_compat_message_callback(int wri >> "quic compat tx %s len:%uz ", >> ngx_quic_level_name(level), len); >> >> - (void) com->method->add_handshake_data(ssl, level, buf, len); >> + if (com->method->add_handshake_data(ssl, level, buf, len) != 1) { >> + goto failed; >> + } >> >> break; >> >> @@ -420,11 +422,19 @@ ngx_quic_compat_message_callback(int wri >> "quic compat %s alert:%ui len:%uz ", >> ngx_quic_level_name(level), alert, len); >> >> - (void) com->method->send_alert(ssl, level, alert); >> + if (com->method->send_alert(ssl, level, alert) != 1) { >> + goto failed; >> + } >> } >> >> break; >> } >> + >> + return; >> + >> +failed: >> + >> + ngx_post_event(&qc->close, &ngx_posted_events); >> } >> > > Looks good! Committed, thanks. ---- Roman Arutyunyan arut at nginx.com -------------- next part -------------- An HTML attachment was scrubbed... URL: From arut at nginx.com Mon Sep 25 14:22:14 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 25 Sep 2023 18:22:14 +0400 Subject: [Proposal] Expose HTTP and Stream proxy modules structures for developers. In-Reply-To: References: Message-ID: <20230925142214.myhuhotvzd3vald5@N00W24XTQX> Hello Andrey, On Sun, Sep 24, 2023 at 02:30:14PM +0300, Andrey Kulikov wrote: > Hello, > > Sometimes custom module developers need to iterate over configured > location(s), configured in some or all server(s) blocks. > For example, to perform post-initialization steps, defined by custom > directives in their modules. > Now it is virtually impossible. > > I've posted question about this matter few years ago: > https://mailman.nginx.org/pipermail/nginx-devel/2016-October/008882.html > > Since then, I received quite a few private emails about whether I succeeded > in my findings. > So, I'm not alone, who faced this issue. You can iterate over locations in a server block using "locations" queue in ngx_http_core_loc_conf_t. You need to use the location configuration at the server level to access the queue. Referencing the example from your last email, it should be something like this: clcf = cscfp[s]->ctx->loc_conf[ngx_http_core_module.ctx_index]; /* iterate over clcf->locations */ See ngx_http_core_location() as an example of how this queue is filled and ngx_http_merge_locations() as an example of how to iterate over it. > Fortunately, the described goal could be easily achieved by making just > internal proxy modules structures available in a separate header. > After that implementation of desired functionality is just a matter of > writing a few lines of code. Accessing proxy module to iterate over locations does not sound like the right decision. > Patches а following. > No functional changes, just code movement from .c to .h files. > > Development community, at least through my voice, will be grateful to see > them applied to the latest development branch. > > -- > Cheers, > Andrey -- Roman Arutyunyan From sdunn934 at gmail.com Mon Sep 25 14:27:40 2023 From: sdunn934 at gmail.com (stewart dunn) Date: Mon, 25 Sep 2023 15:27:40 +0100 Subject: [Proposal] Expose HTTP and Stream proxy modules structures for developers. In-Reply-To: <20230925142214.myhuhotvzd3vald5@N00W24XTQX> References: <20230925142214.myhuhotvzd3vald5@N00W24XTQX> Message-ID: Fyck off On Mon, 25 Sep 2023 at 15:22, Roman Arutyunyan wrote: > Hello Andrey, > > On Sun, Sep 24, 2023 at 02:30:14PM +0300, Andrey Kulikov wrote: > > Hello, > > > > Sometimes custom module developers need to iterate over configured > > location(s), configured in some or all server(s) blocks. > > For example, to perform post-initialization steps, defined by custom > > directives in their modules. > > Now it is virtually impossible. > > > > I've posted question about this matter few years ago: > > https://mailman.nginx.org/pipermail/nginx-devel/2016-October/008882.html > > > > Since then, I received quite a few private emails about whether I > succeeded > > in my findings. > > So, I'm not alone, who faced this issue. > > You can iterate over locations in a server block using "locations" queue in > ngx_http_core_loc_conf_t. You need to use the location configuration at > the server level to access the queue. Referencing the example from your > last > email, it should be something like this: > > clcf = cscfp[s]->ctx->loc_conf[ngx_http_core_module.ctx_index]; > > /* iterate over clcf->locations */ > > See ngx_http_core_location() as an example of how this queue is filled and > ngx_http_merge_locations() as an example of how to iterate over it. > > > Fortunately, the described goal could be easily achieved by making just > > internal proxy modules structures available in a separate header. > > After that implementation of desired functionality is just a matter of > > writing a few lines of code. > > Accessing proxy module to iterate over locations does not sound like the > right decision. > > > Patches а following. > > No functional changes, just code movement from .c to .h files. > > > > Development community, at least through my voice, will be grateful to see > > them applied to the latest development branch. > > > > -- > > Cheers, > > Andrey > > -- > Roman Arutyunyan > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel > -------------- next part -------------- An HTML attachment was scrubbed... URL: From arut at nginx.com Mon Sep 25 14:54:10 2023 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 25 Sep 2023 18:54:10 +0400 Subject: NGINX status code based on *response* body In-Reply-To: References: Message-ID: <20230925145410.cvpetxjgs6vwbosf@N00W24XTQX> Hi Muhammad, On Mon, Sep 25, 2023 at 12:01:11AM +0800, Muhammad Nuzaihan wrote: > Hi, > > I would like to return an error message based on the *response* body. > > I know i can do it easily with *request* body (with "return > NGX_HTTP_BAD_REQUEST") but my case i want to do it with *response* > body. > > Also i have tried this code with response body filter but i got "*1 > header already sent, client: 127.0.0.1, server: localhost": That's because you need to delay sending the response, both header and body, before making the decision about the response code. It's quite complicated. If you send the header right away, there's no way to change it in the future. You can look into ngx_http_xslt_filter_module and ngx_http_image_filter_module for inspiration. For the xslt module (src/http/modules/ngx_http_xslt_filter_module.c), its header filter ngx_http_xslt_header_filter() does not call ngx_http_next_header_filter() on first call (ctx == NULL). Its body filter ngx_http_xslt_body_filter() also does not call ngx_http_next_body_filter() right away. Instead, it buffers the entire response body. Upon body completion, ngx_http_xslt_send() is called, which sends both header (ngx_http_next_header_filter()) and buffered body (ngx_http_next_body_filter()). If a decision is made to trigger an error, ngx_http_filter_finalize_request() is called instead. > Thank you for your guidance. > > Code: > > ``` > return ngx_http_filter_finalize_request(request, > &ngx_m_module, > NGX_HTTP_BAD_REQUEST); > ``` > RESPONSE BODY CODE: > > ngx_int_t ResponseBodyFilter(ngx_http_request_t *request, > ngx_chain_t *chain_head) { > // Set the logging level to debug > // TODO: remove > request->connection->log->log_level = NGX_LOG_DEBUG; > > // Get our context so we can store the response body data > FilterContext *ctx = GetModuleFilterContext(request); > if (ctx == NULL) { > return NGX_ERROR; > } > > return ngx_http_filter_finalize_request(request, > &ngx_my_module, > NGX_HTTP_BAD_REQUEST); > > } > > ngx_int_t ModuleInit(ngx_conf_t *cf) { > kNextRequestBodyFilter = ngx_http_top_request_body_filter; > ngx_http_top_request_body_filter = RequestBodyFilter; > > kNextResponseBodyFilter = ngx_http_top_body_filter; > ngx_http_top_body_filter = ResponseBodyFilter; > > kNextHeaderFilter = ngx_http_top_header_filter; > ngx_http_top_header_filter = HeaderFilter; > > return NGX_OK; > } > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > https://mailman.nginx.org/mailman/listinfo/nginx-devel -- Roman Arutyunyan From mdounin at mdounin.ru Mon Sep 25 19:28:12 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 25 Sep 2023 22:28:12 +0300 Subject: [PATCH 2 of 2] Added "max_shutdown_workers" directive In-Reply-To: <42c3166b675a6a7624bd.1693225844@arut-laptop> References: <42c3166b675a6a7624bd.1693225844@arut-laptop> Message-ID: Hello! On Mon, Aug 28, 2023 at 04:30:44PM +0400, Roman Arutyunyan wrote: > # HG changeset patch > # User Roman Arutyunyan > # Date 1693218974 -14400 > # Mon Aug 28 14:36:14 2023 +0400 > # Node ID 42c3166b675a6a7624bdf3d78c0d4685ce8172e3 > # Parent fdad00808cb485ab83e919be59e9211ef06ac56a > Added "max_shutdown_workers" directive. > > The directive sets maximum number of workers in shutdown state while reloading > nginx configuration. Upon reaching this value, workers restart is postponed > until shutdown workers start exiting. This allows for lower peak resource > consumption at the cost of slower reconfiguration. I can't say I like the name, as the preferred term is "worker processes", not "workers". max_shutdown_processes? Also note that this doesn't really limit the total resource usage during configuration reloads, since cache manager and cache loader processes are exempt from the limit, but new processes are started on each configuration reload. It would be nice to actually provide a limit which makes multiple/frequent configuration reloads completely safe from the resource usage point of view, that is, to limit all processes spawned during configuration reloads, and not just worker processes. > > diff --git a/src/core/nginx.c b/src/core/nginx.c > --- a/src/core/nginx.c > +++ b/src/core/nginx.c > @@ -83,6 +83,13 @@ static ngx_command_t ngx_core_commands[ > 0, > NULL }, > > + { ngx_string("max_shutdown_workers"), > + NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1, > + ngx_conf_set_num_slot, > + 0, > + offsetof(ngx_core_conf_t, max_shutdown_workers), > + NULL }, > + > { ngx_string("debug_points"), > NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1, > ngx_conf_set_enum_slot, > @@ -1117,6 +1124,7 @@ ngx_core_module_create_conf(ngx_cycle_t > ccf->shutdown_timeout = NGX_CONF_UNSET_MSEC; > > ccf->worker_processes = NGX_CONF_UNSET; > + ccf->max_shutdown_workers = NGX_CONF_UNSET; > ccf->debug_points = NGX_CONF_UNSET; > > ccf->rlimit_nofile = NGX_CONF_UNSET; > @@ -1146,6 +1154,7 @@ ngx_core_module_init_conf(ngx_cycle_t *c > ngx_conf_init_msec_value(ccf->shutdown_timeout, 0); > > ngx_conf_init_value(ccf->worker_processes, 1); > + ngx_conf_init_value(ccf->max_shutdown_workers, 0); > ngx_conf_init_value(ccf->debug_points, 0); > > #if (NGX_HAVE_CPU_AFFINITY) > diff --git a/src/core/ngx_cycle.h b/src/core/ngx_cycle.h > --- a/src/core/ngx_cycle.h > +++ b/src/core/ngx_cycle.h > @@ -94,6 +94,7 @@ typedef struct { > ngx_msec_t shutdown_timeout; > > ngx_int_t worker_processes; > + ngx_int_t max_shutdown_workers; > ngx_int_t debug_points; > > ngx_int_t rlimit_nofile; > diff --git a/src/os/unix/ngx_process.c b/src/os/unix/ngx_process.c > --- a/src/os/unix/ngx_process.c > +++ b/src/os/unix/ngx_process.c > @@ -216,6 +216,7 @@ ngx_spawn_process(ngx_cycle_t *cycle, ng > ngx_processes[s].data = data; > ngx_processes[s].name = name; > ngx_processes[s].exiting = 0; > + ngx_processes[s].old = 0; > > switch (respawn) { > > diff --git a/src/os/unix/ngx_process.h b/src/os/unix/ngx_process.h > --- a/src/os/unix/ngx_process.h > +++ b/src/os/unix/ngx_process.h > @@ -33,6 +33,7 @@ typedef struct { > unsigned detached:1; > unsigned exiting:1; > unsigned exited:1; > + unsigned old:1; > } ngx_process_t; > > > diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c > --- a/src/os/unix/ngx_process_cycle.c > +++ b/src/os/unix/ngx_process_cycle.c > @@ -15,6 +15,7 @@ static void ngx_start_worker_processes(n > ngx_int_t type); > static void ngx_start_worker_process(ngx_cycle_t *cycle, ngx_int_t i, > ngx_int_t type); > +static void ngx_restart_worker_processes(ngx_cycle_t *cycle); > static void ngx_start_cache_manager_processes(ngx_cycle_t *cycle, > ngx_uint_t respawn); > static void ngx_pass_open_channel(ngx_cycle_t *cycle); > @@ -22,6 +23,7 @@ static void ngx_signal_worker_processes( > static void ngx_signal_worker_process(ngx_cycle_t *cycle, ngx_int_t i, > int signo); > static ngx_uint_t ngx_reap_children(ngx_cycle_t *cycle); > +static ngx_uint_t ngx_restart_worker_process(ngx_cycle_t *cycle); > static void ngx_master_process_exit(ngx_cycle_t *cycle); > static void ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data); > static void ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker); > @@ -235,8 +237,8 @@ ngx_master_process_cycle(ngx_cycle_t *cy > ngx_cycle = cycle; > ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, > ngx_core_module); > - ngx_start_worker_processes(cycle, ccf->worker_processes, > - NGX_PROCESS_JUST_RESPAWN); > + ngx_restart_worker_processes(cycle); > + > ngx_start_cache_manager_processes(cycle, 1); > > /* allow new processes to start */ > @@ -360,6 +362,70 @@ ngx_start_worker_process(ngx_cycle_t *cy > > > static void > +ngx_restart_worker_processes(ngx_cycle_t *cycle) > +{ > + ngx_int_t i, n, m, worker; > + ngx_core_conf_t *ccf; > + > + ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); > + > + n = 0; > + m = ccf->max_shutdown_workers; > + > + if (m == 0) { > + goto start; > + } > + > + for (i = 0; i < ngx_last_process; i++) { > + > + if (ngx_processes[i].pid == -1 > + || ngx_processes[i].proc != ngx_worker_process_cycle) > + { Checking "proc" here looks like a layering violation. For example, ngx_signal_worker_process() does try to depend on the process entry point, and instead relies on the "detached" flag to decide which processes need to be signalled. > + continue; > + } > + > + ngx_processes[i].old = 0; > + worker = (intptr_t) ngx_processes[i].data; > + > + if (!ngx_processes[i].exiting && worker < ccf->worker_processes) { > + n++; It is not clear what we are counting here, and why. My best guess it that it's an attempt to count worker processes we are going to restart (and not just stop). It is not clear why we are doing this though: the limit as defined does not differentiate between shutting down processes being stopped and processes being restarted. Further, I don't think it's done correctly: since "worker" is an index number of the worker process, there can be at least gaps (if worker process failed to start and cannot be respawn). > + > + } else if (m > 0) { > + m--; > + } > + } > + > + n = (n > m) ? n - m : 0; n = ngx_max(n - m, 0); ? Also, some better names for the variables and/or some comments explaining what is being done might worth the effort. (As far as I can tell, "n" here is the number of processes we want to restart but not allowed to.) > + > + if (n == 0) { > + goto start; > + } > + > + for (i = 0; i < ngx_last_process; i++) { > + > + if (ngx_processes[i].pid == -1 > + || ngx_processes[i].proc != ngx_worker_process_cycle) > + { > + continue; > + } > + > + ngx_processes[i].old = 1; > + worker = (intptr_t) ngx_processes[i].data; > + > + if (!ngx_processes[i].exiting && worker < n) { Here "n" is the number of processes we are not allowed to restart right now, while "worker" is the index number of the worker process in question. For a given "n", number of worker processes matching this condition might be at least less than "n": if some worker processes failed to start. (Just in case, the rule of thumb is: if you are using worker process index number for anything, most likely you are doing it wrong.) > + ngx_processes[i].just_spawn = 1; The "just_spawn" flag implies the process was just started, and so it is ignored by the following ngx_signal_worker_process() call. Abusing it to ignore processes during shutdown might work, but I can't say I like this approach. Using a separate flag might be better. > + } > + } > + > +start: > + > + for (i = n; i < ccf->worker_processes; i++) { > + ngx_start_worker_process(cycle, i, NGX_PROCESS_JUST_RESPAWN); > + } The order of processes being started looks somewhat counter-intuitive here. > +} > + > + > +static void > ngx_start_cache_manager_processes(ngx_cycle_t *cycle, ngx_uint_t respawn) > { > ngx_uint_t i, manager, loader; > @@ -486,15 +552,16 @@ ngx_signal_worker_process(ngx_cycle_t *c > ch.fd = -1; > > > - ngx_log_debug7(NGX_LOG_DEBUG_EVENT, cycle->log, 0, > - "child: %i %P e:%d t:%d d:%d r:%d j:%d", > + ngx_log_debug8(NGX_LOG_DEBUG_EVENT, cycle->log, 0, > + "child: %i %P e:%d t:%d d:%d r:%d j:%d o:%d", > i, > ngx_processes[i].pid, > ngx_processes[i].exiting, > ngx_processes[i].exited, > ngx_processes[i].detached, > ngx_processes[i].respawn, > - ngx_processes[i].just_spawn); > + ngx_processes[i].just_spawn, > + ngx_processes[i].old); > > if (ngx_processes[i].detached || ngx_processes[i].pid == -1) { > return; > @@ -563,15 +630,16 @@ ngx_reap_children(ngx_cycle_t *cycle) > live = 0; > for (i = 0; i < ngx_last_process; i++) { > > - ngx_log_debug7(NGX_LOG_DEBUG_EVENT, cycle->log, 0, > - "child: %i %P e:%d t:%d d:%d r:%d j:%d", > + ngx_log_debug8(NGX_LOG_DEBUG_EVENT, cycle->log, 0, > + "child: %i %P e:%d t:%d d:%d r:%d j:%d o:%d", > i, > ngx_processes[i].pid, > ngx_processes[i].exiting, > ngx_processes[i].exited, > ngx_processes[i].detached, > ngx_processes[i].respawn, > - ngx_processes[i].just_spawn); > + ngx_processes[i].just_spawn, > + ngx_processes[i].old); > > if (ngx_processes[i].pid == -1) { > continue; > @@ -660,6 +728,16 @@ ngx_reap_children(ngx_cycle_t *cycle) > ngx_processes[i].pid = -1; > } > Note: when a process dies, it is restarted if the "respawn" flag is set, regardless of the "old" flag. The "old" flag is preserved even if it is no longer relevant after the respawn: the process is restarted with the new configuration, and there is no need to restart it again. > + if (ngx_processes[i].old > + && ngx_processes[i].exiting > + && !ngx_terminate > + && !ngx_quit) > + { > + if (ngx_restart_worker_process(cycle)) { > + live = 1; > + } > + } > + > } else if (ngx_processes[i].exiting || !ngx_processes[i].detached) { > live = 1; > } > @@ -669,6 +747,53 @@ ngx_reap_children(ngx_cycle_t *cycle) > } > > > +static ngx_uint_t > +ngx_restart_worker_process(ngx_cycle_t *cycle) > +{ > + ngx_int_t i, j, m; > + ngx_core_conf_t *ccf; > + > + ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); > + > + j = -1; > + m = ccf->max_shutdown_workers; > + > + if (m == 0) { > + return 0; > + } > + > + for (i = 0; i < ngx_last_process; i++) { > + > + if (ngx_processes[i].pid == -1 || !ngx_processes[i].old) { > + continue; > + } > + > + if (!ngx_processes[i].exiting && !ngx_processes[i].exited) { The "exited" check here is not needed to catch the processes we were called for, or other processes being shut down - they already have "exiting" flag set. On the other hand, this can catch unexpectedly died processes (the ones which are still waiting for restart). As per the code, such processes will be counted against max_shutdown_workers as if they are shutting down - this looks wrong and can cause reload hang. > + j = i; > + continue; > + } > + > + if (--m == 0) { > + return 0; > + } > + } > + > + if (j == -1) { > + return 0; > + } > + > + ngx_start_worker_process(cycle, (intptr_t) ngx_processes[j].data, > + NGX_PROCESS_RESPAWN); > + > + ngx_msleep(100); > + > + ngx_signal_worker_process(cycle, j, > + ngx_signal_value(NGX_SHUTDOWN_SIGNAL)); Also note that the "exited" check above implies that there can be multiple exited processes at the same time, but we only restart just one process here. It might be a good idea to restart multiple processes at the same time: sleeping for 100ms for each process means that if, for example, 64 processes will exit at the same time after shutdown timeout expiration, master process will be unresponsive for 6 seconds. Also, this looks very close to ngx_restart_worker_processes(). It might be a good idea to rework things to ensure unified approach to restarting processes instead of providing multiple duplicate ways. > + > + return 1; > +} > + > + > static void > ngx_master_process_exit(ngx_cycle_t *cycle) > { Overall, I tend to think that the suggested code is very fragile, and I would rather consider re-thinking the approach. -- Maxim Dounin http://mdounin.ru/ From yar at nginx.com Tue Sep 26 13:14:18 2023 From: yar at nginx.com (=?utf-8?q?Yaroslav_Zhuravlev?=) Date: Tue, 26 Sep 2023 14:14:18 +0100 Subject: [PATCH] Updated OpenSSL and zlib versions Message-ID: <32a78db44cf00ab54063.1695734058@ORK-ML-00007151> xml/en/docs/configure.xml | 6 +++--- xml/en/docs/howto_build_on_win32.xml | 10 +++++----- xml/ru/docs/configure.xml | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) -------------- next part -------------- A non-text attachment was scrubbed... Name: nginx.org.patch Type: text/x-patch Size: 3265 bytes Desc: not available URL: From jt26wzz at gmail.com Tue Sep 26 17:13:44 2023 From: jt26wzz at gmail.com (=?UTF-8?B?5LiK5Yu+5ouz?=) Date: Wed, 27 Sep 2023 01:13:44 +0800 Subject: Memory Leak Issue in Nginx PCRE2 Message-ID: Dear Nginx Developers, I hope this email finds you well. I am reaching out to the mailing list for the first time to report and discuss an issue I encountered while working on supporting PCRE2 in OpenResty. If I have made any errors in my reporting or discussion, please do not hesitate to provide feedback. Your guidance is greatly appreciated. During my recent work, I used the sanitizer to inspect potential issues, and I identified a small memory leak in the PCRE2 code section of Nginx. While this issue does not seem to be critical, it could potentially disrupt memory checking tools. To help you reproduce the problem, I have included a minimal configuration below. Please note that this issue occurs when Nginx is configured to use PCRE2, and the version is 1.22.1 or higher. *Minimal Configuration for Reproduction:* worker_processes 1; daemon off; master_process off; error_log /home/zhenzhongw/code/pcre_pr/lua-nginx-module/t/servroot/logs/error.log debug; pid /home/zhenzhongw/code/pcre_pr/lua-nginx-module/t/servroot/logs/nginx.pid; http { access_log /home/zhenzhongw/code/pcre_pr/lua-nginx-module/t/servroot/logs/access.log; #access_log off; default_type text/plain; keepalive_timeout 68000ms; server { listen 1984; #placeholder server_name 'localhost'; client_max_body_size 30M; #client_body_buffer_size 4k; # Begin preamble config... # End preamble config... # Begin test case config... location ~ '^/[a-d]$' { return 200; } } } events { accept_mutex off; worker_connections 64; } *nginx -V :* nginx version: nginx/1.25.1 (no pool) built by gcc 11.4.1 20230605 (Red Hat 11.4.1-2) (GCC) built with OpenSSL 1.1.1u 30 May 2023 TLS SNI support enabled configure arguments: --prefix=/home/zhenzhongw/code/pcre_pr/lua-nginx-module/work/nginx --with-threads --with-pcre-jit --with-ipv6 --with-cc-opt='-fno-omit-frame-pointer -fsanitize=address -DNGX_LUA_USE_ASSERT -I/opt/pcre2/include -I/opt/ssl/include' --with-http_v2_module --with-http_v3_module --with-http_realip_module --with-http_ssl_module --add-module=/home/zhenzhongw/code/pcre_pr/ndk-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/set-misc-nginx-module --with-ld-opt='-fsanitize=address -L/opt/pcre2/lib -L/opt/ssl/lib -Wl,-rpath,/opt/pcre2/lib:/opt/drizzle/lib:/opt/ssl/lib' --without-mail_pop3_module --without-mail_imap_module --with-http_image_filter_module --without-mail_smtp_module --with-stream --with-stream_ssl_module --without-http_upstream_ip_hash_module --without-http_memcached_module --without-http_auth_basic_module --without-http_userid_module --with-http_auth_request_module --add-module=/home/zhenzhongw/code/pcre_pr/echo-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/memc-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/srcache-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/lua-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/lua-upstream-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/headers-more-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/drizzle-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/rds-json-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/coolkit-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/redis2-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/stream-lua-nginx-module --add-module=/home/zhenzhongw/code/pcre_pr/lua-nginx-module/t/data/fake-module --add-module=/home/zhenzhongw/code/pcre_pr/lua-nginx-module/t/data/fake-shm-module --add-module=/home/zhenzhongw/code/pcre_pr/lua-nginx-module/t/data/fake-delayed-load-module --with-http_gunzip_module --with-http_dav_module --with-select_module --with-poll_module --with-debug --with-poll_module --with-cc=gcc *The sanitizer tool reported the following error message: * ================================================================= ==555798==ERROR: LeakSanitizer: detected memory leaks Direct leak of 72 byte(s) in 1 object(s) allocated from: #0 0x7f502f6b4a07 in __interceptor_malloc (/lib64/libasan.so.6+0xb4a07) #1 0x4a1737 in ngx_alloc src/os/unix/ngx_alloc.c:22 #2 0x525796 in ngx_regex_malloc src/core/ngx_regex.c:509 #3 0x7f502f3e745e in _pcre2_memctl_malloc_8 (/opt/pcre2/lib/libpcre2-8.so.0+0x1145e) #4 0x5771ad in ngx_http_regex_compile src/http/ngx_http_variables.c:2555 #5 0x536088 in ngx_http_core_regex_location src/http/ngx_http_core_module.c:3263 #6 0x537f94 in ngx_http_core_location src/http/ngx_http_core_module.c:3115 #7 0x46ba0a in ngx_conf_handler src/core/ngx_conf_file.c:463 #8 0x46ba0a in ngx_conf_parse src/core/ngx_conf_file.c:319 #9 0x5391ec in ngx_http_core_server src/http/ngx_http_core_module.c:2991 #10 0x46ba0a in ngx_conf_handler src/core/ngx_conf_file.c:463 #11 0x46ba0a in ngx_conf_parse src/core/ngx_conf_file.c:319 #12 0x528e4c in ngx_http_block src/http/ngx_http.c:239 #13 0x46ba0a in ngx_conf_handler src/core/ngx_conf_file.c:463 #14 0x46ba0a in ngx_conf_parse src/core/ngx_conf_file.c:319 #15 0x463f74 in ngx_init_cycle src/core/ngx_cycle.c:284 #12 0x528e4c in ngx_http_block src/http/ngx_http.c:239 #13 0x46ba0a in ngx_conf_handler src/core/ngx_conf_file.c:463 #14 0x46ba0a in ngx_conf_parse src/core/ngx_conf_file.c:319 #15 0x463f74 in ngx_init_cycle src/core/ngx_cycle.c:284 #16 0x4300c7 in main src/core/nginx.c:295 #17 0x7ff31a43feaf in __libc_start_call_main (/lib64/libc.so.6+0x3feaf) SUMMARY: AddressSanitizer: 72 byte(s) leaked in 1 allocation(s). *I have created a patch to address this memory leak issue, which I am sharing below:* diff --git a/src/core/ngx_regex.c b/src/core/ngx_regex.c index 91381f499..71f583789 100644 --- a/src/core/ngx_regex.c +++ b/src/core/ngx_regex.c @@ -600,6 +600,8 @@ ngx_regex_cleanup(void *data) * the new cycle, these will be re-allocated. */ + ngx_regex_malloc_init(NULL); + if (ngx_regex_compile_context) { pcre2_compile_context_free(ngx_regex_compile_context); ngx_regex_compile_context = NULL; @@ -611,6 +613,8 @@ ngx_regex_cleanup(void *data) ngx_regex_match_data_size = 0; } + ngx_regex_malloc_done(); + #endif } @@ -706,7 +710,13 @@ ngx_regex_module_init(ngx_cycle_t *cycle) ngx_regex_malloc_done(); ngx_regex_studies = NULL; + #if (NGX_PCRE2) + if (ngx_regex_compile_context) { + ngx_regex_malloc_init(NULL); + pcre2_compile_context_free(ngx_regex_compile_context); + ngx_regex_malloc_done(); + } ngx_regex_compile_context = NULL; #endif I kindly request your assistance in reviewing this matter and considering the patch for inclusion in Nginx. If you have any questions or need further information, please feel free to reach out to me. Your expertise and feedback are highly valuable in resolving this issue. Thank you for your time and attention to this matter. Best regards, ZhenZhong -------------- next part -------------- An HTML attachment was scrubbed... URL: From xeioex at nginx.com Thu Sep 28 18:01:46 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 28 Sep 2023 18:01:46 +0000 Subject: [njs] Fetch: fixed HEAD response handling with large Content-Length. Message-ID: details: https://hg.nginx.org/njs/rev/a60c0acce58b branches: changeset: 2209:a60c0acce58b user: Dmitry Volyntsev date: Thu Sep 28 11:00:56 2023 -0700 description: Fetch: fixed HEAD response handling with large Content-Length. This closes #658 issue on Github. diffstat: nginx/ngx_js_fetch.c | 5 +++-- nginx/t/js_fetch.t | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diffs (64 lines): diff -r ed935fa4805b -r a60c0acce58b nginx/ngx_js_fetch.c --- a/nginx/ngx_js_fetch.c Fri Sep 22 13:00:06 2023 -0700 +++ b/nginx/ngx_js_fetch.c Thu Sep 28 11:00:56 2023 -0700 @@ -2426,8 +2426,9 @@ ngx_js_http_process_headers(ngx_js_http_ return NGX_ERROR; } - if (hp->content_length_n - > (off_t) http->max_response_body_size) + if (!http->header_only + && hp->content_length_n + > (off_t) http->max_response_body_size) { ngx_js_http_error(http, 0, "fetch content length is too large"); diff -r ed935fa4805b -r a60c0acce58b nginx/t/js_fetch.t --- a/nginx/t/js_fetch.t Fri Sep 22 13:00:06 2023 -0700 +++ b/nginx/t/js_fetch.t Thu Sep 28 11:00:56 2023 -0700 @@ -46,6 +46,8 @@ http { listen 127.0.0.1:8080; server_name localhost; + js_fetch_max_response_buffer_size 128k; + location /njs { js_content test.njs; } @@ -399,7 +401,7 @@ my $p2 = port(8082); host_header, multi, loc, property}; EOF -$t->try_run('no njs.fetch')->plan(35); +$t->try_run('no njs.fetch')->plan(36); $t->run_daemon(\&http_daemon, port(8082)); $t->waitforsocket('127.0.0.1:' . port(8082)); @@ -493,6 +495,13 @@ like(http_get('/host_header?host=FOOBAR' 'fetch host header'); } +TODO: { +local $TODO = 'not yet' unless has_version('0.8.2'); + +like(http_get('/body_special?loc=head/large&method=HEAD'), + qr/200 OK.*$/s, 'fetch head method large content-length'); +} + ############################################################################### sub has_version { @@ -620,6 +629,13 @@ sub http_daemon { "Connection: close" . CRLF . CRLF; + } elsif ($uri eq '/head/large') { + print $client + "HTTP/1.1 200 OK" . CRLF . + "Content-Length: 1000000" . CRLF . + "Connection: close" . CRLF . + CRLF; + } elsif ($uri eq '/parted') { print $client "HTTP/1.1 200 OK" . CRLF . From xeioex at nginx.com Sat Sep 30 00:22:03 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Sat, 30 Sep 2023 00:22:03 +0000 Subject: [njs] Stream: adjusted periodic timeout in tests. Message-ID: details: https://hg.nginx.org/njs/rev/febba7cb31e0 branches: changeset: 2210:febba7cb31e0 user: Dmitry Volyntsev date: Thu Sep 28 18:11:15 2023 -0700 description: Stream: adjusted periodic timeout in tests. diffstat: nginx/t/stream_js_periodic.t | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r a60c0acce58b -r febba7cb31e0 nginx/t/stream_js_periodic.t --- a/nginx/t/stream_js_periodic.t Thu Sep 28 11:00:56 2023 -0700 +++ b/nginx/t/stream_js_periodic.t Thu Sep 28 18:11:15 2023 -0700 @@ -275,7 +275,7 @@ EOF ############################################################################### -select undef, undef, undef, 0.1; +select undef, undef, undef, 0.2; is(stream('127.0.0.1:' . port(8080))->io('affinity'), 'affinity', 'affinity test'); From xeioex at nginx.com Sat Sep 30 00:22:05 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Sat, 30 Sep 2023 00:22:05 +0000 Subject: [njs] RegExp: improved memory footprint when match fails. Message-ID: details: https://hg.nginx.org/njs/rev/88318e18dec9 branches: changeset: 2211:88318e18dec9 user: Dmitry Volyntsev date: Thu Sep 28 19:09:56 2023 -0700 description: RegExp: improved memory footprint when match fails. diffstat: src/njs_regexp.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (33 lines): diff -r febba7cb31e0 -r 88318e18dec9 src/njs_regexp.c --- a/src/njs_regexp.c Thu Sep 28 18:11:15 2023 -0700 +++ b/src/njs_regexp.c Thu Sep 28 19:09:56 2023 -0700 @@ -941,6 +941,9 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj string.size, match_data); if (ret >= 0) { result = njs_regexp_exec_result(vm, r, utf8, &string, match_data); + + njs_regex_match_data_free(match_data, vm->regex_generic_ctx); + if (njs_slow_path(result == NULL)) { return NJS_ERROR; } @@ -949,8 +952,9 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj return NJS_OK; } + njs_regex_match_data_free(match_data, vm->regex_generic_ctx); + if (njs_slow_path(ret == NJS_ERROR)) { - njs_regex_match_data_free(match_data, vm->regex_generic_ctx); return NJS_ERROR; } @@ -1154,8 +1158,6 @@ fail: done: - njs_regex_match_data_free(match_data, vm->regex_generic_ctx); - return (ret == NJS_OK) ? array : NULL; } From mdounin at mdounin.ru Sat Sep 30 03:38:17 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sat, 30 Sep 2023 06:38:17 +0300 Subject: [PATCH] Improve performance when starting nginx with a lot of locations In-Reply-To: References: Message-ID: Hello! On Fri, Sep 22, 2023 at 03:58:41PM +0900, Yusuke Nojima wrote: > # HG changeset patch > # User Yusuke Nojima > # Date 1679555707 -32400 > # Thu Mar 23 16:15:07 2023 +0900 > # Node ID 6aac98fb135e47ca9cf7ad7d780cf4a10e9aa55c > # Parent 8771d35d55d0a2b1cefaab04401d6f837f5a05a2 > Improve performance when starting nginx with a lot of locations > > Our team has a configuration file with a very large number of > locations, and we found that starting nginx with this file takes an > unacceptable amount of time. After investigating the issue, we > discovered that the root cause of the long startup time is the sorting > of the location list. Interesting. Could you please provide some more details about the use case, such as how locations are used, and what is the approximate number of locations being used? In my practice, it is extremely uncommon to use more than 1k-10k prefix locations (and even these numbers are huge for normal setups). Further, since each location contains configuration for all modules, such configurations are expected to require a lot of memory (currently about 5-10KB per empty location, so about 50-100MB per 10k locations, and 0.5-1G per 100k locations). Accordingly, using other approaches such as map (assuming exact match is actually needed) might be beneficial regardless of the sorting costs. Nevertheless, swapping the sorting algorithm to a faster one looks like an obvious improvement. > > Currently, the sorting algorithm used in nginx is insertion sort, > which requires O(n^2) time for n locations. We have modified the > sorting algorithm to use merge sort instead, which has a time > complexity of O(n log n). > > We have tested the modified code using micro-benchmarks and confirmed > that the new algorithm improves nginx startup time significantly > (shown below). We believe that this change would be valuable for other > users who are experiencing similar issues. > > > Table: nginx startup time in seconds > > n current patched > 2000 0.033 0.018 > 4000 0.047 0.028 > 6000 0.062 0.038 > 8000 0.079 0.050 > 10000 0.091 0.065 > 12000 0.367 0.081 > 14000 0.683 0.086 > 16000 0.899 0.097 > 18000 1.145 0.110 > 20000 1.449 0.122 > 22000 1.650 0.137 > 24000 2.155 0.151 > 26000 3.096 0.155 > 28000 3.711 0.168 > 30000 3.539 0.184 > 32000 3.980 0.193 > 34000 4.543 0.208 > 36000 4.349 0.217 > 38000 5.021 0.229 > 40000 4.918 0.245 > 42000 4.835 0.256 > 44000 5.159 0.262 > 46000 5.802 0.331 > 48000 6.205 0.295 > 50000 5.701 0.308 > 52000 5.992 0.335 > 54000 6.561 0.323 > 56000 6.856 0.333 > 58000 6.515 0.347 > 60000 7.051 0.359 > 62000 6.956 0.377 > 64000 7.376 0.376 > 66000 7.506 0.404 > 68000 7.292 0.407 > 70000 7.422 0.461 > 72000 10.090 0.443 > 74000 18.505 0.463 > 76000 11.857 0.462 > 78000 9.752 0.470 > 80000 12.485 0.481 > 82000 11.027 0.498 > 84000 9.804 0.523 > 86000 8.482 0.515 > 88000 9.838 0.560 > 90000 12.341 0.546 > 92000 13.881 0.648 > 94000 8.309 0.635 > 96000 8.854 0.650 > 98000 12.871 0.674 > 100000 8.261 0.698 This probably can be reduced to something like 3-5 data points. > > diff -r 8771d35d55d0 -r 6aac98fb135e src/core/ngx_queue.c > --- a/src/core/ngx_queue.c Fri Mar 10 07:43:50 2023 +0300 > +++ b/src/core/ngx_queue.c Thu Mar 23 16:15:07 2023 +0900 > @@ -45,36 +45,103 @@ > } > > > -/* the stable insertion sort */ > +/* merge queue2 into queue1. queue2 becomes empty after merge. */ > + > +static void > +ngx_queue_merge(ngx_queue_t *queue1, ngx_queue_t *queue2, > + ngx_int_t (*cmp)(const ngx_queue_t *, const ngx_queue_t *)) > +{ > + ngx_queue_t *p1, *p2; Nitpicking: there are various style issues here and in other places. > + > + p1 = ngx_queue_head(queue1); > + p2 = ngx_queue_head(queue2); > + > + while (p1 != ngx_queue_sentinel(queue1) > + && p2 != ngx_queue_sentinel(queue2)) { > + > + if (cmp(p1, p2) > 0) { > + ngx_queue_t *next, *prev; > + > + next = ngx_queue_next(p2); > + ngx_queue_remove(p2); > + prev = ngx_queue_prev(p1); > + ngx_queue_insert_after(prev, p2); > + p2 = next; Nitpicking: there is no need to preserve "next" here, since p2 is always the head of queue2 and, and the next element can be obtained by ngx_queue_head(queue2). Also, instead of obtaining "prev" and using ngx_queue_insert_after() it would be easier to use ngx_queue_insert_before(). It is not currently defined, but it is trivial to define one: it is an alias to ngx_queue_insert_tail(), much like ngx_queue_insert_after() is an alias to ngx_queue_insert_head(). > + } else { > + p1 = ngx_queue_next(p1); > + } > + } > + if (p2 != ngx_queue_sentinel(queue2)) { > + ngx_queue_add(queue1, queue2); > + ngx_queue_init(queue2); > + } > +} > + > + > +/* move all elements from src to dest. dest should be empty before call. */ > + > +static void > +ngx_queue_move(ngx_queue_t *dest, ngx_queue_t *src) > +{ > + *dest = *src; > + ngx_queue_init(src); > + > + if (dest->next == src) { > + dest->next = dest; > + } else { > + dest->next->prev = dest; > + } > + if (dest->prev == src) { > + dest->prev = dest; > + } else { > + dest->prev->next = dest; > + } > +} This function looks strange to me. There is the ngx_queue_add() macro, which probably should be used instead (if needed). > + > + > +/* the stable merge sort */ > > void > ngx_queue_sort(ngx_queue_t *queue, > ngx_int_t (*cmp)(const ngx_queue_t *, const ngx_queue_t *)) > { > - ngx_queue_t *q, *prev, *next; > + ngx_queue_t merged[64], *p, *last; > > - q = ngx_queue_head(queue); > - > - if (q == ngx_queue_last(queue)) { > + if (ngx_queue_head(queue) == ngx_queue_last(queue)) { > return; > } > > - for (q = ngx_queue_next(q); q != ngx_queue_sentinel(queue); q = next) { > + last = merged; > > - prev = ngx_queue_prev(q); > - next = ngx_queue_next(q); > + while (!ngx_queue_empty(queue)) { > + /* > + * Loop invariant: > + * merged[i] must have exactly 0 or 2^i elements in sorted order. > + * For each iteration, we take one element from the given queue and > + * insert it into merged without violating the invariant condition. > + */ > > - ngx_queue_remove(q); > + ngx_queue_t carry, *h; > + > + h = ngx_queue_head(queue); > + ngx_queue_remove(h); > + ngx_queue_init(&carry); > + ngx_queue_insert_head(&carry, h); > > - do { > - if (cmp(prev, q) <= 0) { > - break; > - } > + for (p = merged; p != last && !ngx_queue_empty(p); p++) { > + ngx_queue_merge(p, &carry, cmp); > + ngx_queue_move(&carry, p); > + } > + if (p == last) { > + ngx_queue_init(last); > + last++; > + } > + ngx_queue_move(p, &carry); > + } > > - prev = ngx_queue_prev(prev); > - > - } while (prev != ngx_queue_sentinel(queue)); > - > - ngx_queue_insert_after(prev, q); > + /* Merge all queues into one queue */ > + for (p = merged + 1; p != last; p++) { > + ngx_queue_merge(p, p-1, cmp); > } > + ngx_queue_move(queue, last-1); > } While bottom-up merge sort implementation might be more efficient, I find it disturbing to use fixed array of queues without any checks if we are within the array bounds. Rather, I would suggest recursive top-bottom merge sort implementation instead, which is much simpler and uses stack as temporary storage (so it'll naturally die if there will be a queue which requires more space for sorting than we have). Please take a look if it works for you: diff --git a/src/core/ngx_queue.c b/src/core/ngx_queue.c --- a/src/core/ngx_queue.c +++ b/src/core/ngx_queue.c @@ -9,6 +9,10 @@ #include +static void ngx_queue_merge(ngx_queue_t *queue, ngx_queue_t *tail, + ngx_int_t (*cmp)(const ngx_queue_t *, const ngx_queue_t *)); + + /* * find the middle queue element if the queue has odd number of elements * or the first element of the queue's second part otherwise @@ -45,13 +49,13 @@ ngx_queue_middle(ngx_queue_t *queue) } -/* the stable insertion sort */ +/* the stable merge sort */ void ngx_queue_sort(ngx_queue_t *queue, ngx_int_t (*cmp)(const ngx_queue_t *, const ngx_queue_t *)) { - ngx_queue_t *q, *prev, *next; + ngx_queue_t *q, tail; q = ngx_queue_head(queue); @@ -59,22 +63,44 @@ ngx_queue_sort(ngx_queue_t *queue, return; } - for (q = ngx_queue_next(q); q != ngx_queue_sentinel(queue); q = next) { + q = ngx_queue_middle(queue); + + ngx_queue_split(queue, q, &tail); + + ngx_queue_sort(queue, cmp); + ngx_queue_sort(&tail, cmp); + + ngx_queue_merge(queue, &tail, cmp); +} - prev = ngx_queue_prev(q); - next = ngx_queue_next(q); - ngx_queue_remove(q); +static void +ngx_queue_merge(ngx_queue_t *queue, ngx_queue_t *tail, + ngx_int_t (*cmp)(const ngx_queue_t *, const ngx_queue_t *)) +{ + ngx_queue_t *q1, *q2; + + q1 = ngx_queue_head(queue); + q2 = ngx_queue_head(tail); - do { - if (cmp(prev, q) <= 0) { - break; - } + for ( ;; ) { + if (q1 == ngx_queue_sentinel(queue)) { + ngx_queue_add(queue, tail); + break; + } + + if (q2 == ngx_queue_sentinel(tail)) { + break; + } - prev = ngx_queue_prev(prev); + if (cmp(q1, q2) <= 0) { + q1 = ngx_queue_next(q1); + continue; + } - } while (prev != ngx_queue_sentinel(queue)); + ngx_queue_remove(q2); + ngx_queue_insert_before(q1, q2); - ngx_queue_insert_after(prev, q); + q2 = ngx_queue_head(tail); } } diff --git a/src/core/ngx_queue.h b/src/core/ngx_queue.h --- a/src/core/ngx_queue.h +++ b/src/core/ngx_queue.h @@ -47,6 +47,9 @@ struct ngx_queue_s { (h)->prev = x +#define ngx_queue_insert_before ngx_queue_insert_tail + + #define ngx_queue_head(h) \ (h)->next -- Maxim Dounin http://mdounin.ru/ From xeioex at nginx.com Sat Sep 30 05:43:55 2023 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Sat, 30 Sep 2023 05:43:55 +0000 Subject: [njs] FS: introduced fs.existsSync(). Message-ID: details: https://hg.nginx.org/njs/rev/f076da3c7a6f branches: changeset: 2212:f076da3c7a6f user: Dmitry Volyntsev date: Fri Sep 29 21:41:34 2023 -0700 description: FS: introduced fs.existsSync(). diffstat: external/njs_fs_module.c | 30 ++++++++++++++++++++++++++++++ test/fs/methods.t.js | 31 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 0 deletions(-) diffs (102 lines): diff -r 88318e18dec9 -r f076da3c7a6f external/njs_fs_module.c --- a/external/njs_fs_module.c Thu Sep 28 19:09:56 2023 -0700 +++ b/external/njs_fs_module.c Fri Sep 29 21:41:34 2023 -0700 @@ -146,6 +146,8 @@ typedef njs_int_t (*njs_file_tree_walk_c static njs_int_t njs_fs_access(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t calltype, njs_value_t *retval); +static njs_int_t njs_fs_exists_sync(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t calltype, njs_value_t *retval); static njs_int_t njs_fs_mkdir(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t calltype, njs_value_t *retval); static njs_int_t njs_fs_open(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, @@ -583,6 +585,16 @@ static njs_external_t njs_ext_fs[] = { { .flags = NJS_EXTERN_METHOD, + .name.string = njs_str("existsSync"), + .writable = 1, + .configurable = 1, + .u.method = { + .native = njs_fs_exists_sync, + } + }, + + { + .flags = NJS_EXTERN_METHOD, .name.string = njs_str("fstatSync"), .writable = 1, .configurable = 1, @@ -1470,6 +1482,24 @@ njs_fs_access(njs_vm_t *vm, njs_value_t static njs_int_t +njs_fs_exists_sync(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, + njs_index_t calltype, njs_value_t *retval) +{ + const char *path; + char path_buf[NJS_MAX_PATH + 1]; + + path = njs_fs_path(vm, path_buf, njs_arg(args, nargs, 1), "path"); + if (njs_slow_path(path == NULL)) { + return NJS_ERROR; + } + + njs_value_boolean_set(retval, access(path, F_OK) == 0); + + return NJS_OK; +} + + +static njs_int_t njs_fs_open(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t calltype, njs_value_t *retval) { diff -r 88318e18dec9 -r f076da3c7a6f test/fs/methods.t.js --- a/test/fs/methods.t.js Thu Sep 28 19:09:56 2023 -0700 +++ b/test/fs/methods.t.js Fri Sep 29 21:41:34 2023 -0700 @@ -359,6 +359,36 @@ let appendFileP_tsuite = { get tests() { return append_tests() }, }; +async function exists_test(params) { + let res = await method("exists", params); + + if (res !== params.expected) { + throw Error(`exists failed check`); + } + + return 'SUCCESS'; +} + +let exists_tests = () => [ + { args: ["test/fs/ascii"], + expected: true }, + { args: ["test/fs"], + expected: true }, + { args: ["test/fs_NONEXISTENT/ascii"], + expected: false }, + { args: ["test/fs/ascii_NONEXISTENT"], + expected: false }, +]; + +let existsSync_tsuite = { + name: "fs existsSync", + skip: () => (!has_fs() || !has_buffer()), + T: exists_test, + prepare_args: p, + opts: { type: "sync" }, + get tests() { return exists_tests() }, +}; + async function realpath_test(params) { let data = await method("realpath", params); @@ -1153,6 +1183,7 @@ run([ appendFile_tsuite, appendFileSync_tsuite, appendFileP_tsuite, + existsSync_tsuite, realpath_tsuite, realpathSync_tsuite, realpathP_tsuite, From mdounin at mdounin.ru Sat Sep 30 17:48:59 2023 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sat, 30 Sep 2023 20:48:59 +0300 Subject: [PATCH] Updated OpenSSL and zlib versions In-Reply-To: <32a78db44cf00ab54063.1695734058@ORK-ML-00007151> References: <32a78db44cf00ab54063.1695734058@ORK-ML-00007151> Message-ID: Hello! On Tue, Sep 26, 2023 at 02:14:18PM +0100, Yaroslav Zhuravlev wrote: > xml/en/docs/configure.xml | 6 +++--- > xml/en/docs/howto_build_on_win32.xml | 10 +++++----- > xml/ru/docs/configure.xml | 6 +++--- > 3 files changed, 11 insertions(+), 11 deletions(-) > > > # HG changeset patch > # User Yaroslav Zhuravlev > # Date 1695733917 -3600 > # Tue Sep 26 14:11:57 2023 +0100 > # Node ID 32a78db44cf00ab54063905e2a2ff4e5092a7b28 > # Parent 1ad61bfc7630adf1d6460cf84cec484de4017326 > Updated OpenSSL and zlib versions. > > diff --git a/xml/en/docs/configure.xml b/xml/en/docs/configure.xml > --- a/xml/en/docs/configure.xml > +++ b/xml/en/docs/configure.xml > @@ -8,7 +8,7 @@ >

link="/en/docs/configure.html" > lang="en" > - rev="22"> > + rev="23"> > >

> > @@ -1270,7 +1270,7 @@ > > sets the path to the sources of the zlib library. > The library distribution (version > -1.1.3—1.2.11) needs to be downloaded from the > +1.1.3—1.2.13) needs to be downloaded from the > zlib site and extracted. > The rest is done by nginx’s ./configure and > make. > @@ -1359,7 +1359,7 @@ > --pid-path=/usr/local/nginx/nginx.pid > --with-http_ssl_module > --with-pcre=../pcre2-10.39 > - --with-zlib=../zlib-1.2.11 > + --with-zlib=../zlib-1.2.13 > > > zlib 1.3 was recently released and works just fine, so this probably should be bumped to 1.3 instead. > diff --git a/xml/en/docs/howto_build_on_win32.xml b/xml/en/docs/howto_build_on_win32.xml > --- a/xml/en/docs/howto_build_on_win32.xml > +++ b/xml/en/docs/howto_build_on_win32.xml > @@ -9,7 +9,7 @@ >

link="/en/docs/howto_build_on_win32.html" > lang="en" > - rev="25"> > + rev="26"> > >

> > @@ -81,8 +81,8 @@ > mkdir objs/lib > cd objs/lib > tar -xzf ../../pcre2-10.39.tar.gz > -tar -xzf ../../zlib-1.2.11.tar.gz > -tar -xzf ../../openssl-1.1.1m.tar.gz > +tar -xzf ../../zlib-1.2.13.tar.gz > +tar -xzf ../../openssl-3.0.10.tar.gz > > > > @@ -105,8 +105,8 @@ > --http-uwsgi-temp-path=temp/uwsgi_temp \ > --with-cc-opt=-DFD_SETSIZE=1024 \ > --with-pcre=objs/lib/pcre2-10.39 \ > - --with-zlib=objs/lib/zlib-1.2.11 \ > - --with-openssl=objs/lib/openssl-1.1.1m \ > + --with-zlib=objs/lib/zlib-1.2.13 \ > + --with-openssl=objs/lib/openssl-3.0.10 \ > --with-openssl-opt=no-asm \ > --with-http_ssl_module > And this used to match latest release of nginx/Windows, so zlib 1.2.13 and OpenSSL 3.0.10 are probably fine here. On the other hand, bumping to zlib 1.3 and OpenSSL 3.0.11 might be good enough as well (and will save an update to this file). > diff --git a/xml/ru/docs/configure.xml b/xml/ru/docs/configure.xml > --- a/xml/ru/docs/configure.xml > +++ b/xml/ru/docs/configure.xml > @@ -8,7 +8,7 @@ >

link="/ru/docs/configure.html" > lang="ru" > - rev="22"> > + rev="23"> > >

> > @@ -1250,7 +1250,7 @@ > > задаёт путь к исходным текстам библиотеки zlib. > Дистрибутив библиотеки (версию > -1.1.3—1.2.11) нужно взять на сайте > +1.1.3—1.2.13) нужно взять на сайте > zlib и распаковать. > Всё остальное сделают ./configure nginx’а и > make. > @@ -1339,7 +1339,7 @@ > --pid-path=/usr/local/nginx/nginx.pid > --with-http_ssl_module > --with-pcre=../pcre2-10.39 > - --with-zlib=../zlib-1.2.11 > + --with-zlib=../zlib-1.2.13 > > > Otherwise looks good to me. -- Maxim Dounin http://mdounin.ru/