From cnewton at netflix.com Mon Apr 4 19:44:38 2022 From: cnewton at netflix.com (Chris Newton) Date: Mon, 4 Apr 2022 12:44:38 -0700 Subject: question around ngx_http_get_flushed_variable() Message-ID: I'm a little unclear as to when ngx_http_get_flushed_variable() should be used rather than ngx_http_get_indexed_variable(). The dev guide is a little sparse, so I'm hoping someone could help. The relevant flags within the ngx_variable_value_t with their dev guide descriptions (followed by my interpretation) are: unsigned valid:1; - valid — The value is valid The function used to generate the value has indeed been able to create a valid value. When this is an indexed variable, it is then stored within the ngx_http_request_t.variables array so that it can be reused for this request without needing to be re-generated unsigned not_found:1; - not_found — The variable was not found and thus the data and len fields are irrelevant; this can happen, for example, with variables like $arg_foo when a corresponding argument was not passed in a request This could not be generated, and re-attempting to do so won't succeed. As such (for indexed variables), this is negatively cached within the same ngx_http_request_t.variables array. unsigned no_cacheable:1; - no_cacheable — Do not cache result The difference between functions is that the ngx_http_get_indexed_variable() returns a cached value and ngx_http_get_flushed_variable() flushes the cache for non-cacheable variables. That is, no_cacheable is only meaningful when ngx_http_get_flushed_variable() is called rather than ngx_http_get_indexed_variable(), when it causes the cached value to be regenerated. Are there any guidelines on when ngx_http_get_flushed_variable() should be used instead of ngx_http_get_indexed_variable()? It seems like it is 'practically never', but why for example does ngx_http_get_variable() use ngx_http_get_flushed_variable()? TIA Chris -------------- next part -------------- An HTML attachment was scrubbed... URL: From xeioex at nginx.com Tue Apr 5 03:02:29 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 05 Apr 2022 03:02:29 +0000 Subject: [njs] Tests: introduced OPCODE debug. Message-ID: details: https://hg.nginx.org/njs/rev/6a5ec4a275a6 branches: changeset: 1832:6a5ec4a275a6 user: Dmitry Volyntsev date: Tue Feb 22 19:38:59 2022 +0000 description: Tests: introduced OPCODE debug. diffstat: src/njs_disassembler.c | 18 ++++++------------ src/njs_error.c | 21 ++++++++------------- src/njs_generator.c | 26 ++++++++++++++++++++++---- src/njs_generator.h | 3 ++- src/njs_vm.h | 3 +++ src/njs_vmcode.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 91 insertions(+), 30 deletions(-) diffs (281 lines): diff -r 805c1c96a2d2 -r 6a5ec4a275a6 src/njs_disassembler.c --- a/src/njs_disassembler.c Mon Feb 21 16:53:16 2022 +0000 +++ b/src/njs_disassembler.c Tue Feb 22 19:38:59 2022 +0000 @@ -15,9 +15,6 @@ typedef struct { } njs_code_name_t; -static void njs_disassemble(njs_vm_code_t *code); - - static njs_code_name_t code_names[] = { { NJS_VMCODE_OBJECT, sizeof(njs_vmcode_object_t), @@ -173,7 +170,7 @@ njs_disassembler(njs_vm_t *vm) while (n != 0) { njs_printf("%V:%V\n", &code->file, &code->name); - njs_disassemble(code); + njs_disassemble(code->start, code->end, -1, code->lines); code++; n--; } @@ -182,10 +179,10 @@ njs_disassembler(njs_vm_t *vm) } -static void -njs_disassemble(njs_vm_code_t *code) +void +njs_disassemble(u_char *start, u_char *end, njs_int_t count, njs_arr_t *lines) { - u_char *p, *start, *end; + u_char *p; uint32_t line; njs_str_t *name; njs_uint_t n; @@ -215,9 +212,6 @@ njs_disassemble(njs_vm_code_t *code) njs_vmcode_try_trampoline_t *try_tramp; njs_vmcode_function_frame_t *function; - start = code->start; - end = code->end; - /* * On some 32-bit platform uintptr_t is int and compilers warn * about %l format modifier. size_t has the size as pointer so @@ -226,9 +220,9 @@ njs_disassemble(njs_vm_code_t *code) p = start; - while (p < end) { + while (((p < end) && (count == -1)) || (count-- > 0)) { operation = *(njs_vmcode_operation_t *) p; - line = njs_lookup_line(code, p - start); + line = njs_lookup_line(lines, p - start); if (operation == NJS_VMCODE_ARRAY) { array = (njs_vmcode_array_t *) p; diff -r 805c1c96a2d2 -r 6a5ec4a275a6 src/njs_error.c --- a/src/njs_error.c Mon Feb 21 16:53:16 2022 +0000 +++ b/src/njs_error.c Tue Feb 22 19:38:59 2022 +0000 @@ -1286,7 +1286,6 @@ njs_add_backtrace_entry(njs_vm_t *vm, nj njs_native_frame_t *native_frame) { njs_int_t ret; - njs_uint_t i; njs_vm_code_t *code; njs_function_t *function; njs_backtrace_entry_t *be; @@ -1316,20 +1315,16 @@ njs_add_backtrace_entry(njs_vm_t *vm, nj return NJS_OK; } - code = vm->codes->start; + code = njs_lookup_code(vm, native_frame->pc); - for (i = 0; i < vm->codes->items; i++, code++) { - if (code->start <= native_frame->pc - && native_frame->pc < code->end) - { - be->name = code->name; - be->line = njs_lookup_line(code, native_frame->pc - code->start); - if (!vm->options.quiet) { - be->file = code->file; - } + if (code != NULL) { + be->name = code->name; + be->line = njs_lookup_line(code->lines, native_frame->pc - code->start); + if (!vm->options.quiet) { + be->file = code->file; + } - return NJS_OK; - } + return NJS_OK; } be->name = njs_entry_unknown; diff -r 805c1c96a2d2 -r 6a5ec4a275a6 src/njs_generator.c --- a/src/njs_generator.c Mon Feb 21 16:53:16 2022 +0000 +++ b/src/njs_generator.c Tue Feb 22 19:38:59 2022 +0000 @@ -808,8 +808,26 @@ njs_generate_code_map(njs_vm_t *vm, njs_ } +njs_vm_code_t * +njs_lookup_code(njs_vm_t *vm, u_char *pc) +{ + njs_uint_t i; + njs_vm_code_t *code; + + code = vm->codes->start; + + for (i = 0; i < vm->codes->items; i++, code++) { + if (code->start <= pc && pc < code->end) { + return code; + } + } + + return NULL; +} + + uint32_t -njs_lookup_line(njs_vm_code_t *code, uint32_t offset) +njs_lookup_line(njs_arr_t *lines, uint32_t offset) { njs_uint_t n; njs_vm_line_num_t *map; @@ -817,9 +835,9 @@ njs_lookup_line(njs_vm_code_t *code, uin n = 0; map = NULL; - if (code->lines != NULL) { - n = code->lines->items; - map = (njs_vm_line_num_t *) code->lines->start; + if (lines != NULL) { + n = lines->items; + map = (njs_vm_line_num_t *) lines->start; } while (n != 0) { diff -r 805c1c96a2d2 -r 6a5ec4a275a6 src/njs_generator.h --- a/src/njs_generator.h Mon Feb 21 16:53:16 2022 +0000 +++ b/src/njs_generator.h Tue Feb 22 19:38:59 2022 +0000 @@ -45,7 +45,8 @@ njs_int_t njs_generator_init(njs_generat njs_int_t depth, njs_bool_t runtime); njs_vm_code_t *njs_generate_scope(njs_vm_t *vm, njs_generator_t *generator, njs_parser_scope_t *scope, const njs_str_t *name); -uint32_t njs_lookup_line(njs_vm_code_t *code, uint32_t offset); +njs_vm_code_t *njs_lookup_code(njs_vm_t *vm, u_char *pc); +uint32_t njs_lookup_line(njs_arr_t *lines, uint32_t offset); #endif /* _NJS_GENERATOR_H_INCLUDED_ */ diff -r 805c1c96a2d2 -r 6a5ec4a275a6 src/njs_vm.h --- a/src/njs_vm.h Mon Feb 21 16:53:16 2022 +0000 +++ b/src/njs_vm.h Tue Feb 22 19:38:59 2022 +0000 @@ -258,6 +258,9 @@ njs_int_t njs_builtin_objects_clone(njs_ njs_int_t njs_builtin_match_native_function(njs_vm_t *vm, njs_function_t *function, njs_str_t *name); +void njs_disassemble(u_char *start, u_char *end, njs_int_t count, + njs_arr_t *lines); + njs_arr_t *njs_vm_completions(njs_vm_t *vm, njs_str_t *expression); void *njs_lvlhsh_alloc(void *data, size_t size); diff -r 805c1c96a2d2 -r 6a5ec4a275a6 src/njs_vmcode.c --- a/src/njs_vmcode.c Mon Feb 21 16:53:16 2022 +0000 +++ b/src/njs_vmcode.c Tue Feb 22 19:38:59 2022 +0000 @@ -79,6 +79,11 @@ static njs_jump_off_t njs_function_frame } while (0) +#ifdef NJS_OPCODE_DEBUG +void njs_vmcode_debug(njs_vm_t *vm, u_char *pc, const char *prefix); +#endif + + njs_int_t njs_vmcode_interpreter(njs_vm_t *vm, u_char *pc, void *promise_cap, void *async_ctx) @@ -117,6 +122,10 @@ njs_vmcode_interpreter(njs_vm_t *vm, u_c njs_vmcode_try_trampoline_t *try_trampoline; njs_vmcode_function_frame_t *function_frame; +#ifdef NJS_OPCODE_DEBUG + njs_vmcode_debug(vm, pc, "ENTER"); +#endif + next: for ( ;; ) { @@ -163,6 +172,10 @@ next: * as a single unsigned comparision. */ +#ifdef NJS_OPCODE_DEBUG + njs_disassemble(pc, NULL, 1, NULL); +#endif + if (op > NJS_VMCODE_NORET) { if (op == NJS_VMCODE_MOVE) { @@ -650,6 +663,10 @@ next: njs_vmcode_operand(vm, (njs_index_t) value2, value2); vm->retval = *value2; +#ifdef NJS_OPCODE_DEBUG + njs_vmcode_debug(vm, pc, "EXIT STOP"); +#endif + return NJS_OK; case NJS_VMCODE_JUMP: @@ -723,6 +740,11 @@ next: case NJS_VMCODE_RETURN: njs_vmcode_operand(vm, (njs_index_t) value2, value2); + +#ifdef NJS_OPCODE_DEBUG + njs_vmcode_debug(vm, pc, "EXIT RETURN"); +#endif + return njs_vmcode_return(vm, NULL, value2); case NJS_VMCODE_FUNCTION_COPY: @@ -841,6 +863,11 @@ next: case NJS_VMCODE_AWAIT: await = (njs_vmcode_await_t *) pc; + +#ifdef NJS_OPCODE_DEBUG + njs_vmcode_debug(vm, pc, "EXIT AWAIT"); +#endif + return njs_vmcode_await(vm, await, promise_cap, async_ctx); case NJS_VMCODE_TRY_START: @@ -904,6 +931,11 @@ next: switch (ret) { case NJS_OK: + +#ifdef NJS_OPCODE_DEBUG + njs_vmcode_debug(vm, pc, "EXIT FINALLY"); +#endif + return NJS_OK; case NJS_ERROR: goto error; @@ -1018,6 +1050,10 @@ error: } } +#ifdef NJS_OPCODE_DEBUG + njs_vmcode_debug(vm, pc, "EXIT ERROR"); +#endif + return NJS_ERROR; } @@ -2141,3 +2177,17 @@ njs_vmcode_error(njs_vm_t *vm, u_char *p njs_error_fmt_new(vm, &vm->retval, err->type, "%V", &err->u.message); } } + + +#ifdef NJS_OPCODE_DEBUG +void +njs_vmcode_debug(njs_vm_t *vm, u_char *pc, const char *prefix) +{ + njs_vm_code_t *code; + + code = njs_lookup_code(vm, pc); + + njs_printf("%s %V\n", prefix, + (code != NULL) ? &code->name : &njs_entry_unknown); +} +#endif From xeioex at nginx.com Tue Apr 5 03:02:31 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 05 Apr 2022 03:02:31 +0000 Subject: [njs] Fixed njs_vmcode_interpreter() when "toString" conversion fails. Message-ID: details: https://hg.nginx.org/njs/rev/07ef6c1f04f1 branches: changeset: 1833:07ef6c1f04f1 user: Dmitry Volyntsev date: Mon Mar 28 16:22:17 2022 +0000 description: Fixed njs_vmcode_interpreter() when "toString" conversion fails. Previously, while interpreting a user function, njs_vmcode_interpreter() might return prematurely when an error happens. This is not correct because the current frame has to be unwound (or exception caught) first. The fix is exit through only 5 appropriate exit points to ensure proper unwinding. This closes #467 issue on Github. diffstat: src/njs_vmcode.c | 14 ++++++++------ src/test/njs_unit_test.c | 5 +++++ 2 files changed, 13 insertions(+), 6 deletions(-) diffs (63 lines): diff -r 6a5ec4a275a6 -r 07ef6c1f04f1 src/njs_vmcode.c --- a/src/njs_vmcode.c Tue Feb 22 19:38:59 2022 +0000 +++ b/src/njs_vmcode.c Mon Mar 28 16:22:17 2022 +0000 @@ -700,7 +700,7 @@ next: ret = njs_object_prop_define(vm, value1, &name, function, accessor->type); if (njs_slow_path(ret != NJS_OK)) { - return NJS_ERROR; + goto error; } ret = sizeof(njs_vmcode_prop_accessor_t); @@ -779,12 +779,12 @@ next: } if (njs_slow_path(!njs_is_function(&dst))) { - ret = njs_value_to_key(vm, value2, value2); + ret = njs_value_to_key(vm, &dst, value2); if (njs_slow_path(ret != NJS_OK)) { - return NJS_ERROR; + goto error; } - njs_key_string_get(vm, value2, &string); + njs_key_string_get(vm, &dst, &string); njs_type_error(vm, "(intermediate value)[\"%V\"] is not a function", &string); @@ -950,7 +950,8 @@ next: if (njs_is_valid(value1)) { value1 = njs_mp_alloc(vm->mem_pool, sizeof(njs_value_t)); if (njs_slow_path(value1 == NULL)) { - return NJS_ERROR; + njs_memory_error(vm); + goto error; } njs_scope_value_set(vm, var->dst, value1); @@ -967,7 +968,8 @@ next: value1 = njs_mp_alloc(vm->mem_pool, sizeof(njs_value_t)); if (njs_slow_path(value1 == NULL)) { - return NJS_ERROR; + njs_memory_error(vm); + goto error; } *value1 = *value2; diff -r 6a5ec4a275a6 -r 07ef6c1f04f1 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Tue Feb 22 19:38:59 2022 +0000 +++ b/src/test/njs_unit_test.c Mon Mar 28 16:22:17 2022 +0000 @@ -3409,6 +3409,11 @@ static njs_unit_test_t njs_test[] = /**/ + { njs_str("function f() { Object.prototype.toString = 1; };" + "Object.prototype.toString = f;" + "(function () { try { 's'[{}](); } catch (e) { throw e; } })()"), + njs_str("TypeError: Cannot convert object to primitive value") }, + { njs_str("var i; for (i = 0; i < 10; i++) { i += 1 } i"), njs_str("10") }, From mdounin at mdounin.ru Tue Apr 5 16:02:41 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 5 Apr 2022 19:02:41 +0300 Subject: question around ngx_http_get_flushed_variable() In-Reply-To: References: Message-ID: Hello! On Mon, Apr 04, 2022 at 12:44:38PM -0700, Chris Newton via nginx-devel wrote: > I'm a little unclear as to when ngx_http_get_flushed_variable() should be > used rather than ngx_http_get_indexed_variable(). The dev guide is a little > sparse, so I'm hoping someone could help. [...] > Are there any guidelines on when ngx_http_get_flushed_variable() should be > used instead of ngx_http_get_indexed_variable()? The ngx_http_get_indexed_variable() does not flush non-cacheable variables by itself, and should be used when flushes are not needed and/or handled separately. For example, it is used by the script codes, and this makes it possible to evaluate multiple variables with a single cache flush, such as by ngx_http_script_flush_no_cacheable_variables(). The ngx_http_get_flushed_variable() does a flush automatically when needed, that is, it does not use cache for non-cacheable variables. This is to be used when flushes are not handled separately and might be important. Unless you have some explanation on why using ngx_http_get_indexed_variable() in the particular place is safe, you probably should use ngx_http_get_flushed_variable() instead, or you are risking to end up with incorrect behaviour if you'll try to lookup non-cacheable variables. Just in case, looking through all uses of the ngx_http_get_indexed_variable() function in nginx: - ngx_http_charset_filter_module.c: not sure it is safe, might be a bug; - ngx_http_log_module.c: used with a separate flush by ngx_http_script_flush_no_cacheable_variables(); - ngx_http_memcached_module.c: the $memcached_key variable is expected to be set by "set", so the code assumes no flushes are needed; - ngx_http_userid_filter_module.c: the $uid_reset variables is expected to be set by "set", so the code assumes no flushes are needed. - ngx_http_variables.c: in the ngx_http_get_flushed_variable() function, with automatic flush when needed; - ngx_http_script.c: used when a flush is handled separately. > It seems like it is 'practically never', but why for example does > ngx_http_get_variable() use > > ngx_http_get_flushed_variable()? The ngx_http_get_variable() is to be used for dynamic variables lookup, when you don't know the variable name in advance, notably in SSI and embedded Perl. It is expected to be used without external flushes and so uses ngx_http_get_flushed_variable() to ensure automatic flushes when needed. Hope this helps. -- Maxim Dounin http://mdounin.ru/ From cnewton at netflix.com Wed Apr 6 19:01:02 2022 From: cnewton at netflix.com (Chris Newton) Date: Wed, 6 Apr 2022 12:01:02 -0700 Subject: question around ngx_http_get_flushed_variable() In-Reply-To: References: Message-ID: Thanks Maxim - that does help. I do have a follow up question though, around when to mark variables as no_cacheable. Looking for instance at the request_method variable, it is set via the ngx_http_variable_request_method() function which on success sets v->no_cacheable to 0 - that makes sense to me, since it looks at r->main->method_name and so I would expect it to be fixed for the lifetime of a request However, the declaration of the variable within ngx_http_core_variables has the NGX_HTTP_VAR_NOCACHEABLE flag set. I guess I have two questions here : 1. it seems like having this variable marked as non-cacheable is unnecessary; in this case, there is little work to be done to set it, but I'm trying to understand the criteria being used - I'd imagine that any variable that is not expected to change over the lifetime of the request should be cacheable - in which case variables like msec would be reasonably marked non-cacheable, but I don't see why those like request_method would be? It seems similar in scope to me as http_host which is *not* marked non-cachable 2. is there a reason why the get handler should ever set the variable as non-cacheable, or should it always set it that flag to 0 and instead rely on the call to the ngx_http_add_variable() function to set it? (coding style?) Thank you again for your help Chris On Tue, Apr 5, 2022 at 9:02 AM Maxim Dounin wrote: > Hello! > > On Mon, Apr 04, 2022 at 12:44:38PM -0700, Chris Newton via nginx-devel > wrote: > > > I'm a little unclear as to when ngx_http_get_flushed_variable() should be > > used rather than ngx_http_get_indexed_variable(). The dev guide is a > little > > sparse, so I'm hoping someone could help. > > [...] > > > Are there any guidelines on when ngx_http_get_flushed_variable() should > be > > used instead of ngx_http_get_indexed_variable()? > > The ngx_http_get_indexed_variable() does not flush non-cacheable > variables by itself, and should be used when flushes are not > needed and/or handled separately. For example, it is used by the > script codes, and this makes it possible to evaluate multiple > variables with a single cache flush, such as by > ngx_http_script_flush_no_cacheable_variables(). > > The ngx_http_get_flushed_variable() does a flush automatically > when needed, that is, it does not use cache for non-cacheable > variables. This is to be used when flushes are not handled > separately and might be important. > > Unless you have some explanation on why using > ngx_http_get_indexed_variable() in the particular place is safe, > you probably should use ngx_http_get_flushed_variable() instead, > or you are risking to end up with incorrect behaviour if you'll > try to lookup non-cacheable variables. > > Just in case, looking through all uses of the > ngx_http_get_indexed_variable() function in nginx: > > - ngx_http_charset_filter_module.c: not sure it is safe, might be > a bug; > > - ngx_http_log_module.c: used with a separate flush by > ngx_http_script_flush_no_cacheable_variables(); > > - ngx_http_memcached_module.c: the $memcached_key variable is > expected to be set by "set", so the code assumes no flushes are > needed; > > - ngx_http_userid_filter_module.c: the $uid_reset variables is > expected to be set by "set", so the code assumes no flushes are > needed. > > - ngx_http_variables.c: in the ngx_http_get_flushed_variable() > function, with automatic flush when needed; > > - ngx_http_script.c: used when a flush is handled separately. > > > It seems like it is 'practically never', but why for example does > > ngx_http_get_variable() use > > > > ngx_http_get_flushed_variable()? > > The ngx_http_get_variable() is to be used for dynamic variables > lookup, when you don't know the variable name in advance, notably > in SSI and embedded Perl. It is expected to be used without > external flushes and so uses ngx_http_get_flushed_variable() to > ensure automatic flushes when needed. > > Hope this helps. > > -- > Maxim Dounin > http://mdounin.ru/ > -------------- next part -------------- An HTML attachment was scrubbed... URL: From mdounin at mdounin.ru Wed Apr 6 22:58:22 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 7 Apr 2022 01:58:22 +0300 Subject: question around ngx_http_get_flushed_variable() In-Reply-To: References: Message-ID: Hello! On Wed, Apr 06, 2022 at 12:01:02PM -0700, Chris Newton via nginx-devel wrote: > I do have a follow up question though, around when to mark variables as > no_cacheable. > > Looking for instance at the request_method variable, it is set via the > ngx_http_variable_request_method() > function which on success sets v->no_cacheable to 0 - that makes sense to > me, since it looks at r->main->method_name and so I would expect it to be > fixed for the lifetime of a request > > However, the declaration of the variable within ngx_http_core_variables has > the NGX_HTTP_VAR_NOCACHEABLE flag set. > > I guess I have two questions here : > > 1. it seems like having this variable marked as non-cacheable is > unnecessary; in this case, there is little work to be done to set it, but > I'm trying to understand the criteria being used - I'd imagine that any > variable that is not expected to change over the lifetime of the request > should be cacheable - in which case variables like msec would be reasonably > marked non-cacheable, but I don't see why those like request_method would > be? It seems similar in scope to me as http_host which is *not* marked > non-cachable The $request_method might need to be different for subrequests, and hence non-cacheable. > 2. is there a reason why the get handler should ever set the variable as > non-cacheable, or should it always set it that flag to 0 and instead rely > on the call to the ngx_http_add_variable() function to set it? (coding > style?) As long as the variable is marked as NGX_HTTP_VAR_NOCACHEABLE, it is ok to simply set the flag to 0 (mostly to initialize the bit and simplify the resulting machine code). Though in some cases one may want to mark a particular value returned by the get handler as non-cacheable: for example, $content_length is returned as non-cacheable for chunked request body till the request body is fully read. -- Maxim Dounin http://mdounin.ru/ From xeioex at nginx.com Mon Apr 11 23:18:05 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Mon, 11 Apr 2022 23:18:05 +0000 Subject: [njs] Fixed typo while calculating module path length. Message-ID: details: https://hg.nginx.org/njs/rev/d3ac18fd1cda branches: changeset: 1834:d3ac18fd1cda user: Dmitry Volyntsev date: Mon Apr 11 15:55:14 2022 -0700 description: Fixed typo while calculating module path length. The issue was introduced in 77c398f26d7e (not released yet). diffstat: src/njs_module.c | 2 +- test/js/import_very_long_path.t.js | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletions(-) diffs (25 lines): diff -r 07ef6c1f04f1 -r d3ac18fd1cda src/njs_module.c --- a/src/njs_module.c Mon Mar 28 16:22:17 2022 +0000 +++ b/src/njs_module.c Mon Apr 11 15:55:14 2022 -0700 @@ -118,7 +118,7 @@ njs_module_path(njs_vm_t *vm, const njs_ length = info->name.length; if (dir != NULL) { - length = dir->length; + length += dir->length; if (length == 0) { return NJS_DECLINED; diff -r 07ef6c1f04f1 -r d3ac18fd1cda test/js/import_very_long_path.t.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/js/import_very_long_path.t.js Mon Apr 11 15:55:14 2022 -0700 @@ -0,0 +1,9 @@ +/*--- +: [] +paths: [test/js/module/] +negative: + phase: runtime +---*/ + +import name from 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'; + From xeioex at nginx.com Tue Apr 12 04:27:05 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 12 Apr 2022 04:27:05 +0000 Subject: [njs] Version 0.7.3. Message-ID: details: https://hg.nginx.org/njs/rev/f15d039cf625 branches: changeset: 1835:f15d039cf625 user: Dmitry Volyntsev date: Mon Apr 11 21:22:32 2022 -0700 description: Version 0.7.3. diffstat: CHANGES | 20 ++++++++++++++++++++ 1 files changed, 20 insertions(+), 0 deletions(-) diffs (27 lines): diff -r d3ac18fd1cda -r f15d039cf625 CHANGES --- a/CHANGES Mon Apr 11 15:55:14 2022 -0700 +++ b/CHANGES Mon Apr 11 21:22:32 2022 -0700 @@ -1,3 +1,23 @@ +Changes with njs 0.7.3 12 Apr 2022 + + Core: + + *) Feature: added support of module resolution callback. + This feature allows a host environment to control + how imported modules are loaded. + + *) Bugfix: fixed backtraces while traversing imported user + modules. + + *) Bugfix: fixed Array.prototype.concat() when "this" is a slow + array. + + *) Bugfix: fixed frame allocation from an awaited frame. + + *) Bugfix: fixed allocation of large array literals. + + *) Bugfix: fixed interpreter when "toString" conversion fails. + Changes with njs 0.7.2 25 Jan 2022 Core: From xeioex at nginx.com Tue Apr 12 04:27:07 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 12 Apr 2022 04:27:07 +0000 Subject: [njs] Added tag 0.7.3 for changeset f15d039cf625 Message-ID: details: https://hg.nginx.org/njs/rev/20cbf1730dbb branches: changeset: 1836:20cbf1730dbb user: Dmitry Volyntsev date: Mon Apr 11 21:26:35 2022 -0700 description: Added tag 0.7.3 for changeset f15d039cf625 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r f15d039cf625 -r 20cbf1730dbb .hgtags --- a/.hgtags Mon Apr 11 21:22:32 2022 -0700 +++ b/.hgtags Mon Apr 11 21:26:35 2022 -0700 @@ -48,3 +48,4 @@ dfba7f61745c7454ffdd55303a793206d0a9a84a 8418bd4a4ce3114d57b4d75f913e8c4912bf4b5d 0.7.0 35aca5cc5ea7582b80947caa1e3f4a4fb8ee232d 0.7.1 3dd315b80bab10b6ac475ee25dd207d2eb759881 0.7.2 +f15d039cf625fb92e061f21c9f28a788032a0faa 0.7.3 From vadim.fedorenko at cdnnow.ru Thu Apr 14 23:02:53 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Fri, 15 Apr 2022 02:02:53 +0300 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires Message-ID: # HG changeset patch # User Vadim Fedorenko # Date 1649889268 -10800 # Thu Apr 14 01:34:28 2022 +0300 # Node ID ed7a2c031475bcb252952a467c184c94652b926a # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b Upstream: prioritise Cache-Control over Expires. RFC7234 explicitly says that cache recipient MUST ignore Expires header if response includes Cache-Control header with max-age or s-maxage directives. Previously Cache-Control was ignored if it was after Expires in reply headers. At the same time this patch makes more stable behaviour of using latest value of header Cache-Control even if previous value was 0. Ticket #964 for more information. --- src/http/ngx_http_upstream.c | 25 +++++++++++++++++++++++++ src/http/ngx_http_upstream.h | 15 +++++++++++++++ 2 files changed, 40 insertions(+) diff -r a736a7a613ea -r ed7a2c031475 src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c Tue Feb 08 17:35:27 2022 +0300 +++ b/src/http/ngx_http_upstream.c Thu Apr 14 01:34:28 2022 +0300 @@ -868,6 +868,7 @@ } u->cacheable = 1; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_DEFAULT; c = r->cache; @@ -970,6 +971,7 @@ case NGX_HTTP_CACHE_SCARCE: u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_SCARCE; break; @@ -992,6 +994,7 @@ if (ngx_http_upstream_cache_check_range(r, u) == NGX_DECLINED) { u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_RANGE; } r->cached = 0; @@ -3105,6 +3108,7 @@ case NGX_DECLINED: u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_NO_CACHE; break; default: /* NGX_OK */ @@ -3165,6 +3169,7 @@ } else { u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_INVALID; } } @@ -4690,6 +4695,7 @@ #if (NGX_HTTP_CACHE) if (!(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_SET_COOKIE)) { u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_COOKIE; } #endif @@ -4747,6 +4753,7 @@ || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) { u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_NO_CACHE; return NGX_OK; } @@ -4772,15 +4779,21 @@ } u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_CACHECTRL0; return NGX_OK; } if (n == 0) { u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_CACHECTRL0; return NGX_OK; } r->cache->valid_sec = ngx_time() + n; + u->cacheable_reason &= ~(NGX_HTTP_CACHEABLE_CACHECTRL0|NGX_HTTP_CACHEABLE_EXPIRES); + if (u->cacheable_reason == NGX_HTTP_CACHEABLE_DEFAULT) { + u->cacheable = 1; + } } p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", @@ -4800,9 +4813,13 @@ } u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_CACHECTRL1; return NGX_OK; } + if (u->cacheable_reason == NGX_HTTP_CACHEABLE_DEFAULT) { + u->cacheable = 1; + } r->cache->updating_sec = n; r->cache->error_sec = n; } @@ -4823,10 +4840,15 @@ } u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_CACHECTRL2; return NGX_OK; } r->cache->error_sec = n; + u->cacheable_reason &= ~NGX_HTTP_CACHEABLE_EXPIRES; + if (u->cacheable_reason == NGX_HTTP_CACHEABLE_DEFAULT) { + u->cacheable = 1; + } } } #endif @@ -4864,6 +4886,7 @@ if (expires == NGX_ERROR || expires < ngx_time()) { u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_EXPIRES; return NGX_OK; } @@ -4907,6 +4930,7 @@ switch (n) { case 0: u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_XACCEL; /* fall through */ case NGX_ERROR: @@ -5067,6 +5091,7 @@ || (h->value.len == 1 && h->value.data[0] == '*')) { u->cacheable = 0; + u->cacheable_reason |= NGX_HTTP_CACHEABLE_VARY; } r->cache->vary = h->value; diff -r a736a7a613ea -r ed7a2c031475 src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h Tue Feb 08 17:35:27 2022 +0300 +++ b/src/http/ngx_http_upstream.h Thu Apr 14 01:34:28 2022 +0300 @@ -320,6 +320,20 @@ ngx_http_upstream_t *u); +#define NGX_HTTP_CACHEABLE_DEFAULT 0x80000000 +#define NGX_HTTP_CACHEABLE_SCARCE 0x00000001 +#define NGX_HTTP_CACHEABLE_RANGE 0x00000002 +#define NGX_HTTP_CACHEABLE_INVALID 0x00000004 +#define NGX_HTTP_CACHEABLE_NO_CACHE 0x00000010 +#define NGX_HTTP_CACHEABLE_EXPIRES 0x00000020 +#define NGX_HTTP_CACHEABLE_COOKIE 0x00000040 +#define NGX_HTTP_CACHEABLE_CACHECTRL0 0x00000100 +#define NGX_HTTP_CACHEABLE_CACHECTRL1 0x00000200 +#define NGX_HTTP_CACHEABLE_CACHECTRL2 0x00000400 +#define NGX_HTTP_CACHEABLE_XACCEL 0x00001000 +#define NGX_HTTP_CACHEABLE_VARY 0x00002000 + + struct ngx_http_upstream_s { ngx_http_upstream_handler_pt read_event_handler; ngx_http_upstream_handler_pt write_event_handler; @@ -384,6 +398,7 @@ ngx_http_cleanup_pt *cleanup; + ngx_uint_t cacheable_reason; unsigned store:1; unsigned cacheable:1; unsigned accel:1; From vadim.fedorenko at cdnnow.ru Thu Apr 14 23:03:40 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Fri, 15 Apr 2022 02:03:40 +0300 Subject: [PATCH] Tests: added Expires and Cache-Control headers test Message-ID: <39dea3973d47e0bcd226.1649977420@repo.dev.cdnnow.net> # HG changeset patch # User Vadim Fedorenko # Date 1649976970 -10800 # Fri Apr 15 01:56:10 2022 +0300 # Node ID 39dea3973d47e0bcd226beb3c6554dcdc0e26495 # Parent 0c50a00e67334659d58d3cf7cb81fcf5872a8285 Tests: added Expires and Cache-Control headers test diff -r 0c50a00e6733 -r 39dea3973d47 proxy_cache_expires_cache_control.t --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/proxy_cache_expires_cache_control.t Fri Apr 15 01:56:10 2022 +0300 @@ -0,0 +1,88 @@ +#!/usr/bin/perl + +# (C) Georgii Dzebisashvili + +# Tests for cache management regarding https://trac.nginx.org/nginx/ticket/964 + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; + +use File::Find; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $scriptDir = $FindBin::Bin; +my $cacheDir = $scriptDir."/testcache"; +my $tempDir = $scriptDir."/testtemp"; + +my $t = Test::Nginx->new()->plan(5)->write_file_expand('nginx.conf', <<"EOF"); + +error_log stderr info; +daemon off; +events { + worker_connections 64; +} +http { + access_log off; + proxy_cache_path $cacheDir levels=2 keys_zone=cache_zone:1m inactive=24h max_size=10m; + proxy_temp_path $tempDir; + server { + listen 127.0.0.1:8080; + location / { + proxy_pass http://127.0.0.2:8080; + proxy_cache cache_zone; + } + } + server { + listen 127.0.0.2:8080; + location /success { + add_header Cache-Control "max-age=3600"; + add_header Expires "Tue, 15 Nov 1994 12:45:26 GMT"; + return 200 "Hello world!"; + } + location /fail { + add_header Expires "Tue, 15 Nov 1994 12:45:26 GMT"; + add_header Cache-Control "max-age=3600"; + return 200 "Hello world!"; + } + } +} + + +EOF + +$t->run(); + +############################################################################### + +my $counter = 0; + +sub fileMatchCallback { + -f && $counter++; # Only count files +} + +my $r; + +$r = http_get('/success'); +like($r, qr/Cache-Control/, 'cache-control-first-cache-control-is-present'); +like($r, qr/Expires/, 'cache-control-first-expires-is-present'); + +$r = http_get('/fail'); +like($r, qr/Cache-Control/, 'expires-first-cache-control-is-present'); +like($r, qr/Expires/, 'expires-first-expires-is-present'); + +find(\&fileMatchCallback, $cacheDir); + +is($counter, 2, 'overall number of cached requests'); From xeioex at nginx.com Thu Apr 14 23:08:37 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 14 Apr 2022 23:08:37 +0000 Subject: [njs] Version bump Message-ID: details: https://hg.nginx.org/njs/rev/eab387c72d90 branches: changeset: 1837:eab387c72d90 user: Dmitry Volyntsev date: Thu Apr 14 15:39:15 2022 -0700 description: Version bump diffstat: src/njs.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 20cbf1730dbb -r eab387c72d90 src/njs.h --- a/src/njs.h Mon Apr 11 21:26:35 2022 -0700 +++ b/src/njs.h Thu Apr 14 15:39:15 2022 -0700 @@ -11,7 +11,7 @@ #include -#define NJS_VERSION "0.7.3" +#define NJS_VERSION "0.7.4" #include /* STDOUT_FILENO, STDERR_FILENO */ From xeioex at nginx.com Thu Apr 14 23:08:39 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 14 Apr 2022 23:08:39 +0000 Subject: [njs] Fixed Response headers iteration in Fetch API. Message-ID: details: https://hg.nginx.org/njs/rev/6b226ed1b25d branches: changeset: 1838:6b226ed1b25d user: Dmitry Volyntsev date: Thu Apr 14 16:07:34 2022 -0700 description: Fixed Response headers iteration in Fetch API. Previously, heap-use-after-free might occur when HTTP Response was received with more than 8 headers and headers iteration is used. The fix is not to assume that pointer to the beginning of the keys array never changes. The pointer may change when array is resized. The issue was introduced in 81040de6b085 (0.5.1). This closes #492 issue on Github. diffstat: nginx/ngx_js_fetch.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (15 lines): diff -r eab387c72d90 -r 6b226ed1b25d nginx/ngx_js_fetch.c --- a/nginx/ngx_js_fetch.c Thu Apr 14 15:39:15 2022 -0700 +++ b/nginx/ngx_js_fetch.c Thu Apr 14 16:07:34 2022 -0700 @@ -2234,10 +2234,10 @@ ngx_response_js_ext_keys(njs_vm_t *vm, n length = 0; headers = http->headers.elts; - start = njs_vm_array_start(vm, keys); for (i = 0; i < http->headers.nelts; i++) { h = &headers[i]; + start = njs_vm_array_start(vm, keys); for (k = 0; k < length; k++) { njs_value_string_get(njs_argument(start, k), &hdr); From zhangfei.gao at gmail.com Fri Apr 15 07:58:52 2022 From: zhangfei.gao at gmail.com (Zhangfei Gao) Date: Fri, 15 Apr 2022 15:58:52 +0800 Subject: question about nginx start & stop Message-ID: Hi, I have questions about nginx start and stop I am using // start sudo sbin/nginx //stop sudo sbin/nginx -s quit 1. openssl engine is init (ngx_ssl_init) twice, but openssl engine destroy function is not called. So start nginx and nginx -s quit, engine init twice but not called engine destroy. If we start and stop nginx many times, resource leakage will happen. 2. Currently the nginx master process do ngx_ssl_init then ngx_daemon start daemon and master process exit. Now linux kernel has a patch to release resources in mm_put at process exit. As a result ngx_ssl_init in the master process can not be used in daemon. So is this behavior (release resources in mm_put) not expected? src/core/nginx.c main: ngx_ssl_init OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL) init openssl_engine: bind_fn ngx_daemon start daemon and main process exit Thanks From mdounin at mdounin.ru Sun Apr 17 01:55:04 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sun, 17 Apr 2022 04:55:04 +0300 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: References: Message-ID: Hello! On Fri, Apr 15, 2022 at 02:02:53AM +0300, Vadim Fedorenko via nginx-devel wrote: > # HG changeset patch > # User Vadim Fedorenko > # Date 1649889268 -10800 > # Thu Apr 14 01:34:28 2022 +0300 > # Node ID ed7a2c031475bcb252952a467c184c94652b926a > # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b > Upstream: prioritise Cache-Control over Expires. > > RFC7234 explicitly says that cache recipient MUST ignore Expires > header if response includes Cache-Control header with max-age or > s-maxage directives. Previously Cache-Control was ignored if it > was after Expires in reply headers. > > At the same time this patch makes more stable behaviour of using > latest value of header Cache-Control even if previous value was 0. > Ticket #964 for more information. > --- > src/http/ngx_http_upstream.c | 25 +++++++++++++++++++++++++ > src/http/ngx_http_upstream.h | 15 +++++++++++++++ > 2 files changed, 40 insertions(+) > > diff -r a736a7a613ea -r ed7a2c031475 src/http/ngx_http_upstream.c > --- a/src/http/ngx_http_upstream.c Tue Feb 08 17:35:27 2022 +0300 > +++ b/src/http/ngx_http_upstream.c Thu Apr 14 01:34:28 2022 +0300 > @@ -868,6 +868,7 @@ > } > > u->cacheable = 1; > + u->cacheable_reason |= NGX_HTTP_CACHEABLE_DEFAULT; > > c = r->cache; > > @@ -970,6 +971,7 @@ > case NGX_HTTP_CACHE_SCARCE: > > u->cacheable = 0; > + u->cacheable_reason |= NGX_HTTP_CACHEABLE_SCARCE; > > break; > > @@ -992,6 +994,7 @@ > > if (ngx_http_upstream_cache_check_range(r, u) == NGX_DECLINED) { > u->cacheable = 0; > + u->cacheable_reason |= NGX_HTTP_CACHEABLE_RANGE; > } > > r->cached = 0; > @@ -3105,6 +3108,7 @@ > > case NGX_DECLINED: > u->cacheable = 0; > + u->cacheable_reason |= NGX_HTTP_CACHEABLE_NO_CACHE; > break; > > default: /* NGX_OK */ > @@ -3165,6 +3169,7 @@ > > } else { > u->cacheable = 0; > + u->cacheable_reason |= NGX_HTTP_CACHEABLE_INVALID; > } > } > > @@ -4690,6 +4695,7 @@ > #if (NGX_HTTP_CACHE) > if (!(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_SET_COOKIE)) { > u->cacheable = 0; > + u->cacheable_reason |= NGX_HTTP_CACHEABLE_COOKIE; > } > #endif > > @@ -4747,6 +4753,7 @@ > || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) > { > u->cacheable = 0; > + u->cacheable_reason |= NGX_HTTP_CACHEABLE_NO_CACHE; > return NGX_OK; > } > > @@ -4772,15 +4779,21 @@ > } > > u->cacheable = 0; > + u->cacheable_reason |= NGX_HTTP_CACHEABLE_CACHECTRL0; > return NGX_OK; > } > > if (n == 0) { > u->cacheable = 0; > + u->cacheable_reason |= NGX_HTTP_CACHEABLE_CACHECTRL0; > return NGX_OK; > } > > r->cache->valid_sec = ngx_time() + n; > + u->cacheable_reason &= ~(NGX_HTTP_CACHEABLE_CACHECTRL0|NGX_HTTP_CACHEABLE_EXPIRES); > + if (u->cacheable_reason == NGX_HTTP_CACHEABLE_DEFAULT) { > + u->cacheable = 1; > + } > } > > p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", [...] Thanks for the patch. I'm quite sceptical about attempts to fix this by introducing various flags and reverting cacheable status back to 1. This is not how it should be fixed. -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Sun Apr 17 02:14:09 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sun, 17 Apr 2022 05:14:09 +0300 Subject: question about nginx start & stop In-Reply-To: References: Message-ID: Hello! On Fri, Apr 15, 2022 at 03:58:52PM +0800, Zhangfei Gao wrote: > Hi, > > I have questions about nginx start and stop > I am using > // start > sudo sbin/nginx > //stop > sudo sbin/nginx -s quit > > 1. openssl engine is init (ngx_ssl_init) twice, but openssl engine > destroy function is not called. > So start nginx and nginx -s quit, engine init twice but not called > engine destroy. > If we start and stop nginx many times, resource leakage will happen. OPENSSL_init_ssl manpage says: As of version 1.1.0 OpenSSL will automatically allocate all resources that it needs so no explicit initialisation is required. Similarly it will also automatically deinitialise as required. If there is a resource leak, this is a bug in the OpenSSL engine you are testing with. It's probably up to the OpenSSL development docs how to fix this properly. > 2. Currently the nginx master process do ngx_ssl_init then ngx_daemon > start daemon and master process exit. > Now linux kernel has a patch to release resources in mm_put at process exit. > As a result ngx_ssl_init in the master process can not be used in daemon. > So is this behavior (release resources in mm_put) not expected? > > src/core/nginx.c > main: > ngx_ssl_init > OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL) > init openssl_engine: bind_fn > ngx_daemon > start daemon and main process exit Forked processes are expected to match the original process on Unix systems in terms of available resources, such as allocated memory and open files. If in your setup OpenSSL library becomes unusable after fork(), expect multiple issues. -- Maxim Dounin http://mdounin.ru/ From vadim.fedorenko at cdnnow.ru Sun Apr 17 19:50:25 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Sun, 17 Apr 2022 20:50:25 +0100 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: References: Message-ID: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> Hi Maxim! On 17.04.2022 02:55, Maxim Dounin wrote: > Hello! > > > [...] > > Thanks for the patch. > > I'm quite sceptical about attempts to fix this by introducing > various flags and reverting cacheable status back to 1. This is > not how it should be fixed. > Yeah, the solution might be a bit complicated, but I couldn't find another way without breaking concept of independent header parsing. Could you please suggest something if you think that current approach is wrong? The ticket that this patch tries to fix is 6 years old and still has discussions going on without any solution. Thanks, Vadim From yugo-horie at jocdn.co.jp Mon Apr 18 01:14:37 2022 From: yugo-horie at jocdn.co.jp (Yugo Horie) Date: Mon, 18 Apr 2022 10:14:37 +0900 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> Message-ID: We're also interested in this issue because the CDN provider like us is concerned about the problems with the headers that determine the cache freshness. It is often a serious problem used in Nginx as the origin server or the intermediate cache server that has such problems. In those days, we also proposed a patch to resolve it. https://mailman.nginx.org/archives/list/nginx-devel at nginx.org/thread/ZQVPGEXAAHVL23TBBNMXLP4KDGDJCVTW/ But it seems to have declined. We sincerely wish to resolve it and intend to do anything. Thanks, Yugo Horie 2022年4月18日(月) 4:51 Vadim Fedorenko via nginx-devel : > Hi Maxim! > > On 17.04.2022 02:55, Maxim Dounin wrote: > > Hello! > > > > > > [...] > > > > Thanks for the patch. > > > > I'm quite sceptical about attempts to fix this by introducing > > various flags and reverting cacheable status back to 1. This is > > not how it should be fixed. > > > Yeah, the solution might be a bit complicated, but I couldn't find > another way without breaking concept of independent header parsing. > Could you please suggest something if you think that current approach > is wrong? The ticket that this patch tries to fix is 6 years old and > still has discussions going on without any solution. > > Thanks, > Vadim > _______________________________________________ > nginx-devel mailing list -- nginx-devel at nginx.org > To unsubscribe send an email to nginx-devel-leave at nginx.org > -------------- next part -------------- An HTML attachment was scrubbed... URL: From vadim.fedorenko at cdnnow.ru Mon Apr 18 10:19:55 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Mon, 18 Apr 2022 11:19:55 +0100 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> Message-ID: <1910add5-24f9-7109-5c4f-5a71c2ebb24e@cdnnow.ru> Hi Yugo! Looks like another solution with the same concept - store additional information about several headers. The second patch you proposed looks a bit overcomplicated inside this additional function. I don't see any reasons to store information about "Vary" header. AFAIU, weed special information about 3 headers: "Expires", "Cache-Control" and "X-Accel-Expires". And we can call "ngx_http_upstream_cache_validate_regardless_order" only when cacheable is still enabled after parsing all other headers. That approach will simplify the logic inside ngx_http_upstream_cache_validate_regardless_order. I will prepare an improved version if you don't mind. Thanks, Vadim On 18.04.2022 02:14, Yugo Horie wrote: > We're also interested in this issue because the CDN provider like us is > concerned about the problems with the headers that determine the cache freshness. > It is often a serious problem used in Nginx as the origin server or the > intermediate cache server that has such problems. > In those days, we also proposed a patch to resolve it. > https://mailman.nginx.org/archives/list/nginx-devel at nginx.org/thread/ZQVPGEXAAHVL23TBBNMXLP4KDGDJCVTW/ > > > But it seems to have declined. We sincerely wish to resolve it and intend to do > anything. > > Thanks, > Yugo Horie > > 2022年4月18日(月) 4:51 Vadim Fedorenko via nginx-devel >: > > Hi Maxim! > > On 17.04.2022 02:55, Maxim Dounin wrote: > > Hello! > > > > > > [...] > > > > Thanks for the patch. > > > > I'm quite sceptical about attempts to fix this by introducing > > various flags and reverting cacheable status back to 1.  This is > > not how it should be fixed. > > > Yeah, the solution might be a bit complicated, but I couldn't find > another way without breaking concept of independent header parsing. > Could you please suggest something if you think that current approach > is wrong? The ticket that this patch tries to fix is 6 years old and > still has discussions going on without any solution. > > Thanks, > Vadim > _______________________________________________ > nginx-devel mailing list -- nginx-devel at nginx.org > To unsubscribe send an email to nginx-devel-leave at nginx.org > > From yugo-horie at jocdn.co.jp Mon Apr 18 23:03:29 2022 From: yugo-horie at jocdn.co.jp (Yugo Horie) Date: Tue, 19 Apr 2022 08:03:29 +0900 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: <1910add5-24f9-7109-5c4f-5a71c2ebb24e@cdnnow.ru> References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> <1910add5-24f9-7109-5c4f-5a71c2ebb24e@cdnnow.ru> Message-ID: Hi, Vadim. Thanks for your cooperation! > @@ -5067,6 +5091,7 @@ > || (h->value.len == 1 && h->value.data[0] == '*')) > { > u->cacheable = 0; > + u->cacheable_reason |= NGX_HTTP_CACHEABLE_VARY; > } > I don't see any reasons to store information about "Vary" header. It should not cache only the Vary asterisk. it doesn't need to store this information so we consider that your approach (bitmask) is correct. > And we can call > "ngx_http_upstream_cache_validate_regardless_order" only when cacheable is still > enabled after parsing all other headers. Our approach is a bit different, u->cacheable should be determined after completing all headers parsing. So we consider that it could be better not to set here and there during parsing headers particularly in case of u->cacheable reverts 1 to 0 if there is, it could be misread easily. > I will prepare an improved version if you don't mind. Sure! Can we pick your brain for this issue? Thanks, Yugo Horie On Mon, Apr 18, 2022 at 19:19 Vadim Fedorenko wrote: > Hi Yugo! > > Looks like another solution with the same concept - store additional > information > about several headers. The second patch you proposed looks a bit > overcomplicated > inside this additional function. I don't see any reasons to store > information > about "Vary" header. AFAIU, weed special information about 3 headers: > "Expires", > "Cache-Control" and "X-Accel-Expires". And we can call > "ngx_http_upstream_cache_validate_regardless_order" only when cacheable is > still > enabled after parsing all other headers. That approach will simplify the > logic > inside ngx_http_upstream_cache_validate_regardless_order. I will prepare an > improved version if you don't mind. > > Thanks, > Vadim > > On 18.04.2022 02:14, Yugo Horie wrote: > > We're also interested in this issue because the CDN provider like us is > > concerned about the problems with the headers that determine the cache > freshness. > > It is often a serious problem used in Nginx as the origin server or the > > intermediate cache server that has such problems. > > In those days, we also proposed a patch to resolve it. > > > https://mailman.nginx.org/archives/list/nginx-devel at nginx.org/thread/ZQVPGEXAAHVL23TBBNMXLP4KDGDJCVTW/ > > < > https://mailman.nginx.org/archives/list/nginx-devel at nginx.org/thread/ZQVPGEXAAHVL23TBBNMXLP4KDGDJCVTW/ > > > > > > But it seems to have declined. We sincerely wish to resolve it and > intend to do > > anything. > > > > Thanks, > > Yugo Horie > > > > 2022年4月18日(月) 4:51 Vadim Fedorenko via nginx-devel < > nginx-devel at nginx.org > > >: > > > > Hi Maxim! > > > > On 17.04.2022 02:55, Maxim Dounin wrote: > > > Hello! > > > > > > > > > [...] > > > > > > Thanks for the patch. > > > > > > I'm quite sceptical about attempts to fix this by introducing > > > various flags and reverting cacheable status back to 1. This is > > > not how it should be fixed. > > > > > Yeah, the solution might be a bit complicated, but I couldn't find > > another way without breaking concept of independent header parsing. > > Could you please suggest something if you think that current approach > > is wrong? The ticket that this patch tries to fix is 6 years old and > > still has discussions going on without any solution. > > > > Thanks, > > Vadim > > _______________________________________________ > > nginx-devel mailing list -- nginx-devel at nginx.org nginx-devel at nginx.org> > > To unsubscribe send an email to nginx-devel-leave at nginx.org > > > > > -------------- next part -------------- An HTML attachment was scrubbed... URL: From zhangfei.gao at gmail.com Tue Apr 19 04:13:45 2022 From: zhangfei.gao at gmail.com (Zhangfei Gao) Date: Tue, 19 Apr 2022 12:13:45 +0800 Subject: question about nginx start & stop In-Reply-To: References: Message-ID: Hi, Maxim Thanks for the reply. On Sun, Apr 17, 2022 at 10:14 AM Maxim Dounin wrote: > > Hello! > > On Fri, Apr 15, 2022 at 03:58:52PM +0800, Zhangfei Gao wrote: > > > Hi, > > > > I have questions about nginx start and stop > > I am using > > // start > > sudo sbin/nginx > > //stop > > sudo sbin/nginx -s quit > > > > 1. openssl engine is init (ngx_ssl_init) twice, but openssl engine > > destroy function is not called. > > So start nginx and nginx -s quit, engine init twice but not called > > engine destroy. > > If we start and stop nginx many times, resource leakage will happen. > > OPENSSL_init_ssl manpage says: > > As of version 1.1.0 OpenSSL will automatically allocate all resources > that it needs so no explicit initialisation is required. Similarly it > will also automatically deinitialise as required. > > If there is a resource leak, this is a bug in the OpenSSL engine > you are testing with. It's probably up to the OpenSSL development > docs how to fix this properly. The openssl engine is registered with IMPLEMENT_DYNAMIC_BIND_FN(bind_fn) bind_fn() { ENGINE_set_destroy_function(e, destroy) ENGINE_set_finish_function(e, finish) } What I found is. /sbin/nginx -> bind_fn /sbin/nginx -s quit -> bind_fn So bind_fn is called twice, but destroy and finish are not called at all. src/core/nginx.c main ngx_ssl_init(log) -> call engine: bind_fn if (ngx_signal) return ngx_signal_process(cycle, ngx_signal); -> gx_os_signal_process(cycle, sig, pid) // send signal to kill worker process For openssl engine, what should I do to match the nginx stop. By the way, bind_fn and destroy are matched if testing with openssl application itself, no leakage. Any suggestions? Thanks > > > 2. Currently the nginx master process do ngx_ssl_init then ngx_daemon > > start daemon and master process exit. > > Now linux kernel has a patch to release resources in mm_put at process exit. > > As a result ngx_ssl_init in the master process can not be used in daemon. > > So is this behavior (release resources in mm_put) not expected? > > > > src/core/nginx.c > > main: > > ngx_ssl_init > > OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL) > > init openssl_engine: bind_fn > > ngx_daemon > > start daemon and main process exit > > Forked processes are expected to match the original process on > Unix systems in terms of available resources, such as allocated > memory and open files. If in your setup OpenSSL library becomes > unusable after fork(), expect multiple issues. > > -- > Maxim Dounin > http://mdounin.ru/ > _______________________________________________ > nginx-devel mailing list -- nginx-devel at nginx.org > To unsubscribe send an email to nginx-devel-leave at nginx.org From mdounin at mdounin.ru Tue Apr 19 14:01:01 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 19 Apr 2022 17:01:01 +0300 Subject: question about nginx start & stop In-Reply-To: References: Message-ID: Hello! On Tue, Apr 19, 2022 at 12:13:45PM +0800, Zhangfei Gao wrote: > Hi, Maxim > > Thanks for the reply. > > On Sun, Apr 17, 2022 at 10:14 AM Maxim Dounin wrote: > > > > Hello! > > > > On Fri, Apr 15, 2022 at 03:58:52PM +0800, Zhangfei Gao wrote: > > > > > Hi, > > > > > > I have questions about nginx start and stop > > > I am using > > > // start > > > sudo sbin/nginx > > > //stop > > > sudo sbin/nginx -s quit > > > > > > 1. openssl engine is init (ngx_ssl_init) twice, but openssl engine > > > destroy function is not called. > > > So start nginx and nginx -s quit, engine init twice but not called > > > engine destroy. > > > If we start and stop nginx many times, resource leakage will happen. > > > > OPENSSL_init_ssl manpage says: > > > > As of version 1.1.0 OpenSSL will automatically allocate all resources > > that it needs so no explicit initialisation is required. Similarly it > > will also automatically deinitialise as required. > > > > If there is a resource leak, this is a bug in the OpenSSL engine > > you are testing with. It's probably up to the OpenSSL development > > docs how to fix this properly. > > The openssl engine is registered with > IMPLEMENT_DYNAMIC_BIND_FN(bind_fn) > bind_fn() > { > ENGINE_set_destroy_function(e, destroy) > ENGINE_set_finish_function(e, finish) > } > > What I found is. > /sbin/nginx > -> bind_fn > /sbin/nginx -s quit > -> bind_fn > So bind_fn is called twice, but destroy and finish are not called at all. > > src/core/nginx.c > main > ngx_ssl_init(log) -> call engine: bind_fn > if (ngx_signal) > return ngx_signal_process(cycle, ngx_signal); > -> gx_os_signal_process(cycle, sig, pid) // send signal to kill > worker process > > For openssl engine, what should I do to match the nginx stop. > > By the way, bind_fn and destroy are matched if testing with openssl > application itself, no leakage. > > Any suggestions? The bind function is called when loading a dynamic engine and is not expected to be matched by neither destroy nor finish. The finish function is called when releasing a functional reference obtained with init (ENGINE_init() + ENGINE_finish()). The destroy function is called when releasing a structural reference (ENGINE_new() + ENGINE_free()). You shouldn't allocate resources in the bind function, but rather only set appropriate init function to do so when needed. -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Tue Apr 19 15:01:51 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 19 Apr 2022 18:01:51 +0300 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> Message-ID: Hello! On Sun, Apr 17, 2022 at 08:50:25PM +0100, Vadim Fedorenko via nginx-devel wrote: > On 17.04.2022 02:55, Maxim Dounin wrote: > > > I'm quite sceptical about attempts to fix this by introducing > > various flags and reverting cacheable status back to 1. This is > > not how it should be fixed. > > Yeah, the solution might be a bit complicated, but I couldn't find > another way without breaking concept of independent header parsing. > Could you please suggest something if you think that current approach > is wrong? The ticket that this patch tries to fix is 6 years old and > still has discussions going on without any solution. That's basically why the ticket is still not fixed: there is no easy fix, and the issue is rather minor, especially compared to the complexity of the changes required to fix it. IMHO, the most promising approach would be to move Cache-Control/Exires/X-Accel-Expires handling elsewhere, and handle them somewhere in ngx_http_upstream_process_headers(), probably along with ngx_http_file_cache_valid() as well. -- Maxim Dounin http://mdounin.ru/ From vadim.fedorenko at cdnnow.ru Tue Apr 19 22:20:06 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Wed, 20 Apr 2022 01:20:06 +0300 Subject: [PATCH v2] Upstream: prioritise X-Accel-Expire over Cache-Control and Expire Message-ID: # HG changeset patch # User Vadim Fedorenko # Date 1650406016 -10800 # Wed Apr 20 01:06:56 2022 +0300 # Node ID a75449f4b5c98df0e5a0041eeaa1be5c82d92fea # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b Upstream: prioritise X-Accel-Expire over Cache-Control and Expire RFC7234 explicitly says that cache recipient MUST ignore Expires header if response includes Cache-Control header with max-age or s-maxage directives. Previously Cache-Control was ignored if it was after Expires in reply headers. At the same time documentation states that there is special header X-Accel-Expires that controls the behavior of nginx and should be prioritized over other cache-specific headers and this patch implements this priority. More informantion could be found in ticket #964. --- src/http/ngx_http_upstream.c | 62 +++++++++++++++++++++++++----------- src/http/ngx_http_upstream.h | 6 ++++ 2 files changed, 50 insertions(+), 18 deletions(-) diff -r a736a7a613ea -r a75449f4b5c9 src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c Tue Feb 08 17:35:27 2022 +0300 +++ b/src/http/ngx_http_upstream.c Wed Apr 20 01:06:56 2022 +0300 @@ -2350,6 +2350,32 @@ static void +ngx_http_upstream_validate_cache_headers(ngx_http_request_t *r, ngx_http_upstream_t *u) +{ + ngx_http_upstream_headers_in_t *uh = &u->headers_in; + if (uh->x_accel_expires != NULL && + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_EXPIRES)) { + u->cacheable = uh->x_accel_expires_c; + r->cache->valid_sec = uh->x_accel_expires_n; + return; + } + + if (uh->cache_control.elts != NULL && + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_CACHE_CONTROL)) { + u->cacheable = uh->cache_control_c; + r->cache->valid_sec = uh->cache_control_n; + return; + } + + if (uh->expires != NULL && + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_EXPIRES)) { + u->cacheable = uh->expires_c; + r->cache->valid_sec = uh->expires_n; + } +} + + +static void ngx_http_upstream_process_header(ngx_http_request_t *r, ngx_http_upstream_t *u) { ssize_t n; @@ -2468,6 +2494,11 @@ continue; } +#if (NGX_HTTP_CACHE) + if (u->cacheable) { + ngx_http_upstream_validate_cache_headers(r, u); + } +#endif break; } @@ -4735,10 +4766,6 @@ return NGX_OK; } - if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { - return NGX_OK; - } - start = h->value.data; last = start + h->value.len; @@ -4746,7 +4773,7 @@ || ngx_strlcasestrn(start, last, (u_char *) "no-store", 8 - 1) != NULL || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) { - u->cacheable = 0; + u->headers_in.cache_control_c = 0; return NGX_OK; } @@ -4771,7 +4798,6 @@ continue; } - u->cacheable = 0; return NGX_OK; } @@ -4780,7 +4806,8 @@ return NGX_OK; } - r->cache->valid_sec = ngx_time() + n; + u->headers_in.cache_control_c = 1; + u->headers_in.cache_control_n = ngx_time() + n; } p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", @@ -4856,18 +4883,18 @@ return NGX_OK; } - if (r->cache->valid_sec != 0) { + expires = ngx_parse_http_time(h->value.data, h->value.len); + + if (expires == NGX_ERROR || expires < ngx_time()) { return NGX_OK; } - expires = ngx_parse_http_time(h->value.data, h->value.len); - - if (expires == NGX_ERROR || expires < ngx_time()) { - u->cacheable = 0; - return NGX_OK; - } - - r->cache->valid_sec = expires; + if (u->headers_in.expires_c) { + expires = ngx_min(expires, u->headers_in.expires_n); + } + u->headers_in.expires_n = expires; + u->headers_in.expires_c = 1; + } #endif @@ -4906,14 +4933,12 @@ switch (n) { case 0: - u->cacheable = 0; - /* fall through */ - case NGX_ERROR: return NGX_OK; default: - r->cache->valid_sec = ngx_time() + n; + u->headers_in.x_accel_expires_c = 1; + u->headers_in.x_accel_expires_n = ngx_time() + n; return NGX_OK; } } @@ -4924,7 +4949,8 @@ n = ngx_atoi(p, len); if (n != NGX_ERROR) { - r->cache->valid_sec = n; + u->headers_in.x_accel_expires_c = 1; + u->headers_in.x_accel_expires_n = ngx_time() + n; } } #endif diff -r a736a7a613ea -r a75449f4b5c9 src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h Tue Feb 08 17:35:27 2022 +0300 +++ b/src/http/ngx_http_upstream.h Wed Apr 20 01:06:56 2022 +0300 @@ -294,9 +294,15 @@ off_t content_length_n; time_t last_modified_time; + ngx_int_t cache_control_n; + ngx_int_t expires_n; + ngx_int_t x_accel_expires_n; unsigned connection_close:1; unsigned chunked:1; + unsigned cache_control_c; + unsigned expires_c; + unsigned x_accel_expires_c; } ngx_http_upstream_headers_in_t; From vadim.fedorenko at cdnnow.ru Tue Apr 19 22:20:42 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Wed, 20 Apr 2022 01:20:42 +0300 Subject: [PATCH v2] Tests: added Expires and Cache-Control headers test Message-ID: <3d5684530a8ef228cd7f.1650406842@repo.dev.cdnnow.net> # HG changeset patch # User Vadim Fedorenko # Date 1649976970 -10800 # Fri Apr 15 01:56:10 2022 +0300 # Node ID 3d5684530a8ef228cd7f20ff3e51f9ea5e77f2ec # Parent 0c50a00e67334659d58d3cf7cb81fcf5872a8285 Tests: added Expires and Cache-Control headers test diff -r 0c50a00e6733 -r 3d5684530a8e proxy_cache_expires_cache_control.t --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/proxy_cache_expires_cache_control.t Fri Apr 15 01:56:10 2022 +0300 @@ -0,0 +1,99 @@ +#!/usr/bin/perl + +# (C) Georgii Dzebisashvili +# (C) Vadim Fedorenko + +# Tests for cache management regarding https://trac.nginx.org/nginx/ticket/964 + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; + +use File::Find; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $scriptDir = $FindBin::Bin; +my $cacheDir = $scriptDir."/testcache"; +my $tempDir = $scriptDir."/testtemp"; + +my $t = Test::Nginx->new()->plan(8)->write_file_expand('nginx.conf', <<"EOF"); + +daemon off; +events { + worker_connections 64; +} +http { + access_log off; + proxy_cache_path $cacheDir levels=2 keys_zone=cache_zone:1m inactive=24h max_size=10m; + proxy_temp_path $tempDir; + server { + listen 127.0.0.1:8080; + location / { + proxy_pass http://127.0.0.2:8080; + proxy_cache cache_zone; + } + } + server { + listen 127.0.0.2:8080; + location /success { + add_header Cache-Control "max-age=3600"; + add_header Expires "Tue, 15 Nov 1994 12:45:26 GMT"; + return 200 "Hello world!"; + } + location /fail { + add_header Expires "Tue, 15 Nov 1994 12:45:26 GMT"; + add_header Cache-Control "max-age=3600"; + return 200 "Hello world!"; + } + location /store_xaccelexpire { + add_header Expires "Tue, 15 Nov 1994 12:45:26 GMT"; + add_header Cache-Control "no-cache"; + add_header X-Accel-Expires "60"; + return 200 "Hello world!"; + } + } +} + + +EOF + +$t->run(); + +############################################################################### + +my $counter = 0; + +sub fileMatchCallback { + -f && $counter++; # Only count files +} + +my $r; + +$r = http_get('/success'); +like($r, qr/Cache-Control/, 'cache-control-first-cache-control-is-present'); +like($r, qr/Expires/, 'cache-control-first-expires-is-present'); + +$r = http_get('/fail'); +like($r, qr/Cache-Control/, 'expires-first-cache-control-is-present'); +like($r, qr/Expires/, 'expires-first-expires-is-present'); + +$r = http_get('/store_xaccelexpire'); +like($r, qr/Cache-Control/, 'cache-control-with-xaccel-is-present'); +like($r, qr/Expires/, 'expires-with-xaccel-is-present'); +unlike($r, qr/X-Accel-Expires/, 'xaccel-is-not-present'); + +find(\&fileMatchCallback, $cacheDir); + +is($counter, 3, 'overall number of cached requests'); From yugo-horie at jocdn.co.jp Wed Apr 20 04:22:03 2022 From: yugo-horie at jocdn.co.jp (Yugo Horie) Date: Wed, 20 Apr 2022 13:22:03 +0900 Subject: [PATCH v2] Upstream: prioritise X-Accel-Expire over Cache-Control and Expire In-Reply-To: References: Message-ID: Hi, Vadim We're pleased to have you share your patch! src/http/ngx_http_upstream.c ``` 4803 if (n == 0) { 4804 u->cacheable = 0; 4805 return NGX_OK; 4806 } ``` We consider that X-Accel-Expires seems not to prioritize Cache-Control:max-age=0, when it parses after Cache-Control:max-age=0. Because u->cacheable has been 0 according to this code it cannot apply the X-Accel-Expires values with `ngx_http_upstream_validate_cache_headers`. Could we set u->headers_in.cache_control_c to 0 or only return NGX_OK to do nothing here instead of u->cacheable set 0? Best, Yugo Horie 2022年4月20日(水) 7:20 Vadim Fedorenko via nginx-devel : > > # HG changeset patch > # User Vadim Fedorenko > # Date 1650406016 -10800 > # Wed Apr 20 01:06:56 2022 +0300 > # Node ID a75449f4b5c98df0e5a0041eeaa1be5c82d92fea > # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b > Upstream: prioritise X-Accel-Expire over Cache-Control and Expire > > RFC7234 explicitly says that cache recipient MUST ignore Expires > header if response includes Cache-Control header with max-age or > s-maxage directives. Previously Cache-Control was ignored if it > was after Expires in reply headers. > > At the same time documentation states that there is special header > X-Accel-Expires that controls the behavior of nginx and should be > prioritized over other cache-specific headers and this patch > implements this priority. > > More informantion could be found in ticket #964. > --- > src/http/ngx_http_upstream.c | 62 +++++++++++++++++++++++++----------- > src/http/ngx_http_upstream.h | 6 ++++ > 2 files changed, 50 insertions(+), 18 deletions(-) > > diff -r a736a7a613ea -r a75449f4b5c9 src/http/ngx_http_upstream.c > --- a/src/http/ngx_http_upstream.c Tue Feb 08 17:35:27 2022 +0300 > +++ b/src/http/ngx_http_upstream.c Wed Apr 20 01:06:56 2022 +0300 > @@ -2350,6 +2350,32 @@ > > > static void > +ngx_http_upstream_validate_cache_headers(ngx_http_request_t *r, ngx_http_upstream_t *u) > +{ > + ngx_http_upstream_headers_in_t *uh = &u->headers_in; > + if (uh->x_accel_expires != NULL && > + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_EXPIRES)) { > + u->cacheable = uh->x_accel_expires_c; > + r->cache->valid_sec = uh->x_accel_expires_n; > + return; > + } > + > + if (uh->cache_control.elts != NULL && > + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_CACHE_CONTROL)) { > + u->cacheable = uh->cache_control_c; > + r->cache->valid_sec = uh->cache_control_n; > + return; > + } > + > + if (uh->expires != NULL && > + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_EXPIRES)) { > + u->cacheable = uh->expires_c; > + r->cache->valid_sec = uh->expires_n; > + } > +} > + > + > +static void > ngx_http_upstream_process_header(ngx_http_request_t *r, ngx_http_upstream_t *u) > { > ssize_t n; > @@ -2468,6 +2494,11 @@ > > continue; > } > +#if (NGX_HTTP_CACHE) > + if (u->cacheable) { > + ngx_http_upstream_validate_cache_headers(r, u); > + } > +#endif > > break; > } > @@ -4735,10 +4766,6 @@ > return NGX_OK; > } > > - if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { > - return NGX_OK; > - } > - > start = h->value.data; > last = start + h->value.len; > > @@ -4746,7 +4773,7 @@ > || ngx_strlcasestrn(start, last, (u_char *) "no-store", 8 - 1) != NULL > || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) > { > - u->cacheable = 0; > + u->headers_in.cache_control_c = 0; > return NGX_OK; > } > > @@ -4771,7 +4798,6 @@ > continue; > } > > - u->cacheable = 0; > return NGX_OK; > } > > @@ -4780,7 +4806,8 @@ > return NGX_OK; > } > > - r->cache->valid_sec = ngx_time() + n; > + u->headers_in.cache_control_c = 1; > + u->headers_in.cache_control_n = ngx_time() + n; > } > > p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", > @@ -4856,18 +4883,18 @@ > return NGX_OK; > } > > - if (r->cache->valid_sec != 0) { > + expires = ngx_parse_http_time(h->value.data, h->value.len); > + > + if (expires == NGX_ERROR || expires < ngx_time()) { > return NGX_OK; > } > > - expires = ngx_parse_http_time(h->value.data, h->value.len); > - > - if (expires == NGX_ERROR || expires < ngx_time()) { > - u->cacheable = 0; > - return NGX_OK; > - } > - > - r->cache->valid_sec = expires; > + if (u->headers_in.expires_c) { > + expires = ngx_min(expires, u->headers_in.expires_n); > + } > + u->headers_in.expires_n = expires; > + u->headers_in.expires_c = 1; > + > } > #endif > > @@ -4906,14 +4933,12 @@ > > switch (n) { > case 0: > - u->cacheable = 0; > - /* fall through */ > - > case NGX_ERROR: > return NGX_OK; > > default: > - r->cache->valid_sec = ngx_time() + n; > + u->headers_in.x_accel_expires_c = 1; > + u->headers_in.x_accel_expires_n = ngx_time() + n; > return NGX_OK; > } > } > @@ -4924,7 +4949,8 @@ > n = ngx_atoi(p, len); > > if (n != NGX_ERROR) { > - r->cache->valid_sec = n; > + u->headers_in.x_accel_expires_c = 1; > + u->headers_in.x_accel_expires_n = ngx_time() + n; > } > } > #endif > diff -r a736a7a613ea -r a75449f4b5c9 src/http/ngx_http_upstream.h > --- a/src/http/ngx_http_upstream.h Tue Feb 08 17:35:27 2022 +0300 > +++ b/src/http/ngx_http_upstream.h Wed Apr 20 01:06:56 2022 +0300 > @@ -294,9 +294,15 @@ > > off_t content_length_n; > time_t last_modified_time; > + ngx_int_t cache_control_n; > + ngx_int_t expires_n; > + ngx_int_t x_accel_expires_n; > > unsigned connection_close:1; > unsigned chunked:1; > + unsigned cache_control_c; > + unsigned expires_c; > + unsigned x_accel_expires_c; > } ngx_http_upstream_headers_in_t; > > > _______________________________________________ > nginx-devel mailing list -- nginx-devel at nginx.org > To unsubscribe send an email to nginx-devel-leave at nginx.org From vadim.fedorenko at cdnnow.ru Wed Apr 20 09:18:24 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Wed, 20 Apr 2022 10:18:24 +0100 Subject: [PATCH v2] Upstream: prioritise X-Accel-Expire over Cache-Control and Expire In-Reply-To: References: Message-ID: Hi Yugo, Looks like you are right, I will post v3 with updates soon. Thanks, Vadim On 20.04.2022 05:22, Yugo Horie wrote: > Hi, Vadim > We're pleased to have you share your patch! > > src/http/ngx_http_upstream.c > ``` > 4803 if (n == 0) { > 4804 u->cacheable = 0; > 4805 return NGX_OK; > 4806 } > ``` > We consider that X-Accel-Expires seems not to prioritize > Cache-Control:max-age=0, when it parses after Cache-Control:max-age=0. > Because u->cacheable has been 0 according to this code it cannot apply > the X-Accel-Expires values with > `ngx_http_upstream_validate_cache_headers`. > Could we set u->headers_in.cache_control_c to 0 or only return NGX_OK > to do nothing here instead of u->cacheable set 0? > > Best, > > Yugo Horie > > > 2022年4月20日(水) 7:20 Vadim Fedorenko via nginx-devel : >> >> # HG changeset patch >> # User Vadim Fedorenko >> # Date 1650406016 -10800 >> # Wed Apr 20 01:06:56 2022 +0300 >> # Node ID a75449f4b5c98df0e5a0041eeaa1be5c82d92fea >> # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b >> Upstream: prioritise X-Accel-Expire over Cache-Control and Expire >> >> RFC7234 explicitly says that cache recipient MUST ignore Expires >> header if response includes Cache-Control header with max-age or >> s-maxage directives. Previously Cache-Control was ignored if it >> was after Expires in reply headers. >> >> At the same time documentation states that there is special header >> X-Accel-Expires that controls the behavior of nginx and should be >> prioritized over other cache-specific headers and this patch >> implements this priority. >> >> More informantion could be found in ticket #964. >> --- >> src/http/ngx_http_upstream.c | 62 +++++++++++++++++++++++++----------- >> src/http/ngx_http_upstream.h | 6 ++++ >> 2 files changed, 50 insertions(+), 18 deletions(-) >> >> diff -r a736a7a613ea -r a75449f4b5c9 src/http/ngx_http_upstream.c >> --- a/src/http/ngx_http_upstream.c Tue Feb 08 17:35:27 2022 +0300 >> +++ b/src/http/ngx_http_upstream.c Wed Apr 20 01:06:56 2022 +0300 >> @@ -2350,6 +2350,32 @@ >> >> >> static void >> +ngx_http_upstream_validate_cache_headers(ngx_http_request_t *r, ngx_http_upstream_t *u) >> +{ >> + ngx_http_upstream_headers_in_t *uh = &u->headers_in; >> + if (uh->x_accel_expires != NULL && >> + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_EXPIRES)) { >> + u->cacheable = uh->x_accel_expires_c; >> + r->cache->valid_sec = uh->x_accel_expires_n; >> + return; >> + } >> + >> + if (uh->cache_control.elts != NULL && >> + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_CACHE_CONTROL)) { >> + u->cacheable = uh->cache_control_c; >> + r->cache->valid_sec = uh->cache_control_n; >> + return; >> + } >> + >> + if (uh->expires != NULL && >> + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_EXPIRES)) { >> + u->cacheable = uh->expires_c; >> + r->cache->valid_sec = uh->expires_n; >> + } >> +} >> + >> + >> +static void >> ngx_http_upstream_process_header(ngx_http_request_t *r, ngx_http_upstream_t *u) >> { >> ssize_t n; >> @@ -2468,6 +2494,11 @@ >> >> continue; >> } >> +#if (NGX_HTTP_CACHE) >> + if (u->cacheable) { >> + ngx_http_upstream_validate_cache_headers(r, u); >> + } >> +#endif >> >> break; >> } >> @@ -4735,10 +4766,6 @@ >> return NGX_OK; >> } >> >> - if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { >> - return NGX_OK; >> - } >> - >> start = h->value.data; >> last = start + h->value.len; >> >> @@ -4746,7 +4773,7 @@ >> || ngx_strlcasestrn(start, last, (u_char *) "no-store", 8 - 1) != NULL >> || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) >> { >> - u->cacheable = 0; >> + u->headers_in.cache_control_c = 0; >> return NGX_OK; >> } >> >> @@ -4771,7 +4798,6 @@ >> continue; >> } >> >> - u->cacheable = 0; >> return NGX_OK; >> } >> >> @@ -4780,7 +4806,8 @@ >> return NGX_OK; >> } >> >> - r->cache->valid_sec = ngx_time() + n; >> + u->headers_in.cache_control_c = 1; >> + u->headers_in.cache_control_n = ngx_time() + n; >> } >> >> p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", >> @@ -4856,18 +4883,18 @@ >> return NGX_OK; >> } >> >> - if (r->cache->valid_sec != 0) { >> + expires = ngx_parse_http_time(h->value.data, h->value.len); >> + >> + if (expires == NGX_ERROR || expires < ngx_time()) { >> return NGX_OK; >> } >> >> - expires = ngx_parse_http_time(h->value.data, h->value.len); >> - >> - if (expires == NGX_ERROR || expires < ngx_time()) { >> - u->cacheable = 0; >> - return NGX_OK; >> - } >> - >> - r->cache->valid_sec = expires; >> + if (u->headers_in.expires_c) { >> + expires = ngx_min(expires, u->headers_in.expires_n); >> + } >> + u->headers_in.expires_n = expires; >> + u->headers_in.expires_c = 1; >> + >> } >> #endif >> >> @@ -4906,14 +4933,12 @@ >> >> switch (n) { >> case 0: >> - u->cacheable = 0; >> - /* fall through */ >> - >> case NGX_ERROR: >> return NGX_OK; >> >> default: >> - r->cache->valid_sec = ngx_time() + n; >> + u->headers_in.x_accel_expires_c = 1; >> + u->headers_in.x_accel_expires_n = ngx_time() + n; >> return NGX_OK; >> } >> } >> @@ -4924,7 +4949,8 @@ >> n = ngx_atoi(p, len); >> >> if (n != NGX_ERROR) { >> - r->cache->valid_sec = n; >> + u->headers_in.x_accel_expires_c = 1; >> + u->headers_in.x_accel_expires_n = ngx_time() + n; >> } >> } >> #endif >> diff -r a736a7a613ea -r a75449f4b5c9 src/http/ngx_http_upstream.h >> --- a/src/http/ngx_http_upstream.h Tue Feb 08 17:35:27 2022 +0300 >> +++ b/src/http/ngx_http_upstream.h Wed Apr 20 01:06:56 2022 +0300 >> @@ -294,9 +294,15 @@ >> >> off_t content_length_n; >> time_t last_modified_time; >> + ngx_int_t cache_control_n; >> + ngx_int_t expires_n; >> + ngx_int_t x_accel_expires_n; >> >> unsigned connection_close:1; >> unsigned chunked:1; >> + unsigned cache_control_c; >> + unsigned expires_c; >> + unsigned x_accel_expires_c; >> } ngx_http_upstream_headers_in_t; >> >> >> _______________________________________________ >> nginx-devel mailing list -- nginx-devel at nginx.org >> To unsubscribe send an email to nginx-devel-leave at nginx.org From vadim.fedorenko at cdnnow.ru Wed Apr 20 09:18:43 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Wed, 20 Apr 2022 12:18:43 +0300 Subject: [PATCH v3] Upstream: prioritise X-Accel-Expire over Cache-Control and Expire Message-ID: # HG changeset patch # User Vadim Fedorenko # Date 1650406016 -10800 # Wed Apr 20 01:06:56 2022 +0300 # Node ID e04dac22328020cf8d8abcc4863b982b513b0c80 # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b Upstream: prioritise X-Accel-Expire over Cache-Control and Expire RFC7234 explicitly says that cache recipient MUST ignore Expires header if response includes Cache-Control header with max-age or s-maxage directives. Previously Cache-Control was ignored if it was after Expires in reply headers. At the same time documentation states that there is special header X-Accel-Expires that controls the behavior of nginx and should be prioritized over other cache-specific headers and this patch implements this priority. More informantion could be found in ticket #964. --- src/http/ngx_http_upstream.c | 62 +++++++++++++++++++++++++----------- src/http/ngx_http_upstream.h | 6 ++++ 2 files changed, 50 insertions(+), 18 deletions(-) diff -r a736a7a613ea -r e04dac223280 src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c Tue Feb 08 17:35:27 2022 +0300 +++ b/src/http/ngx_http_upstream.c Wed Apr 20 01:06:56 2022 +0300 @@ -2350,6 +2350,32 @@ static void +ngx_http_upstream_validate_cache_headers(ngx_http_request_t *r, ngx_http_upstream_t *u) +{ + ngx_http_upstream_headers_in_t *uh = &u->headers_in; + if (uh->x_accel_expires != NULL && + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_EXPIRES)) { + u->cacheable = uh->x_accel_expires_c; + r->cache->valid_sec = uh->x_accel_expires_n; + return; + } + + if (uh->cache_control.elts != NULL && + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_CACHE_CONTROL)) { + u->cacheable = uh->cache_control_c; + r->cache->valid_sec = uh->cache_control_n; + return; + } + + if (uh->expires != NULL && + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_EXPIRES)) { + u->cacheable = uh->expires_c; + r->cache->valid_sec = uh->expires_n; + } +} + + +static void ngx_http_upstream_process_header(ngx_http_request_t *r, ngx_http_upstream_t *u) { ssize_t n; @@ -2468,6 +2494,11 @@ continue; } +#if (NGX_HTTP_CACHE) + if (u->cacheable) { + ngx_http_upstream_validate_cache_headers(r, u); + } +#endif break; } @@ -4735,10 +4766,6 @@ return NGX_OK; } - if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { - return NGX_OK; - } - start = h->value.data; last = start + h->value.len; @@ -4746,7 +4773,7 @@ || ngx_strlcasestrn(start, last, (u_char *) "no-store", 8 - 1) != NULL || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) { - u->cacheable = 0; + u->headers_in.cache_control_c = 0; return NGX_OK; } @@ -4771,16 +4798,15 @@ continue; } - u->cacheable = 0; return NGX_OK; } if (n == 0) { - u->cacheable = 0; return NGX_OK; } - r->cache->valid_sec = ngx_time() + n; + u->headers_in.cache_control_c = 1; + u->headers_in.cache_control_n = ngx_time() + n; } p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", @@ -4856,18 +4882,18 @@ return NGX_OK; } - if (r->cache->valid_sec != 0) { + expires = ngx_parse_http_time(h->value.data, h->value.len); + + if (expires == NGX_ERROR || expires < ngx_time()) { return NGX_OK; } - expires = ngx_parse_http_time(h->value.data, h->value.len); - - if (expires == NGX_ERROR || expires < ngx_time()) { - u->cacheable = 0; - return NGX_OK; - } - - r->cache->valid_sec = expires; + if (u->headers_in.expires_c) { + expires = ngx_min(expires, u->headers_in.expires_n); + } + u->headers_in.expires_n = expires; + u->headers_in.expires_c = 1; + } #endif @@ -4906,14 +4932,12 @@ switch (n) { case 0: - u->cacheable = 0; - /* fall through */ - case NGX_ERROR: return NGX_OK; default: - r->cache->valid_sec = ngx_time() + n; + u->headers_in.x_accel_expires_c = 1; + u->headers_in.x_accel_expires_n = ngx_time() + n; return NGX_OK; } } @@ -4924,7 +4948,8 @@ n = ngx_atoi(p, len); if (n != NGX_ERROR) { - r->cache->valid_sec = n; + u->headers_in.x_accel_expires_c = 1; + u->headers_in.x_accel_expires_n = ngx_time() + n; } } #endif diff -r a736a7a613ea -r e04dac223280 src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h Tue Feb 08 17:35:27 2022 +0300 +++ b/src/http/ngx_http_upstream.h Wed Apr 20 01:06:56 2022 +0300 @@ -294,9 +294,15 @@ off_t content_length_n; time_t last_modified_time; + ngx_int_t cache_control_n; + ngx_int_t expires_n; + ngx_int_t x_accel_expires_n; unsigned connection_close:1; unsigned chunked:1; + unsigned cache_control_c; + unsigned expires_c; + unsigned x_accel_expires_c; } ngx_http_upstream_headers_in_t; From mdounin at mdounin.ru Wed Apr 20 22:18:41 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:41 +0300 Subject: [PATCH 01 of 20] Perl: fixed $r->header_in("Connection") In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1650492314 -10800 # Thu Apr 21 01:05:14 2022 +0300 # Node ID e70fb0fdfbc0fb7b7e9f493cc2eb65de617b115a # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b Perl: fixed $r->header_in("Connection"). Previously, the r->header_in->connection pointer was never set despite being present in ngx_http_headers_in, resulting in incorrect value returned by $r->header_in("Connection") in embedded perl. diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -1827,6 +1827,10 @@ static ngx_int_t ngx_http_process_connection(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { + if (ngx_http_process_header_line(r, h, offset) != NGX_OK) { + return NGX_ERROR; + } + if (ngx_strcasestrn(h->value.data, "close", 5 - 1)) { r->headers_in.connection_type = NGX_HTTP_CONNECTION_CLOSE; From mdounin at mdounin.ru Wed Apr 20 22:18:40 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:40 +0300 Subject: [PATCH 00 of 20] multiple headers handling Message-ID: Hello! The following patch series is expected to resolve issues with multiple headers with identical names. In particular, the following Trac tickets are fixed: https://trac.nginx.org/nginx/ticket/1724 Nginx doesn't sanitize and is inconsistent with multiple, repeated input headers https://trac.nginx.org/nginx/ticket/1316 $http_ variables only contain the first field-value https://trac.nginx.org/nginx/ticket/1843 $upstream_http_set_cookie includes only first cookie https://trac.nginx.org/nginx/ticket/1423 response vary headers not used in the cache key https://trac.nginx.org/nginx/ticket/485 Multiple WWW-Authenticate headers The basic idea is that all headers are linked lists now, which makes it possible to easily concatenate multiple headers without introducing arrays, as it was previously done with Cookie and X-Forwarded-For request headers and Cache-Control, Link, and Set-Cookie response headers. Futher, such approach makes it possible to easily concatenate unknown headers at runtime, so it makes it possible to properly merge headers passed to CGI-like backends. Review and testing appreciated. -- Maxim Dounin From mdounin at mdounin.ru Wed Apr 20 22:18:43 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:43 +0300 Subject: [PATCH 03 of 20] SCGI: combining headers with identical names (ticket #1724) In-Reply-To: References: Message-ID: <99c3ac43b556830c90ed.1650493123@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492318 -10800 # Thu Apr 21 01:05:18 2022 +0300 # Node ID 99c3ac43b556830c90ed4e8cf179d429a4830112 # Parent 61b29233a55216c6fa72e23b93a4a28d76a9fb94 SCGI: combining headers with identical names (ticket #1724). SCGI specification explicitly forbids headers with duplicate names (section "3. Request Format"): "Duplicate names are not allowed in the headers". Further, provided headers are expected to follow CGI specification, which also requires to combine headers (RFC 3875, section "4.1.18. Protocol-Specific Meta-Variables"): "If multiple header fields with the same field-name are received then the server MUST rewrite them as a single value having the same semantics". diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c --- a/src/http/modules/ngx_http_scgi_module.c +++ b/src/http/modules/ngx_http_scgi_module.c @@ -633,14 +633,14 @@ static ngx_int_t ngx_http_scgi_create_request(ngx_http_request_t *r) { off_t content_length_n; - u_char ch, *key, *val, *lowcase_key; + u_char ch, sep, *key, *val, *lowcase_key; size_t len, key_len, val_len, allocated; ngx_buf_t *b; ngx_str_t content_length; ngx_uint_t i, n, hash, skip_empty, header_params; ngx_chain_t *cl, *body; ngx_list_part_t *part; - ngx_table_elt_t *header, **ignored; + ngx_table_elt_t *header, *hn, **ignored; ngx_http_scgi_params_t *params; ngx_http_script_code_pt code; ngx_http_script_engine_t e, le; @@ -707,7 +707,11 @@ ngx_http_scgi_create_request(ngx_http_re allocated = 0; lowcase_key = NULL; - if (params->number) { + if (ngx_http_link_multi_headers(r) != NGX_OK) { + return NGX_ERROR; + } + + if (params->number || r->headers_in.multi) { n = 0; part = &r->headers_in.headers.part; @@ -737,6 +741,12 @@ ngx_http_scgi_create_request(ngx_http_re i = 0; } + for (n = 0; n < header_params; n++) { + if (&header[i] == ignored[n]) { + goto next_length; + } + } + if (params->number) { if (allocated < header[i].key.len) { allocated = header[i].key.len + 16; @@ -770,6 +780,15 @@ ngx_http_scgi_create_request(ngx_http_re len += sizeof("HTTP_") - 1 + header[i].key.len + 1 + header[i].value.len + 1; + + for (hn = header[i].next; hn; hn = hn->next) { + len += hn->value.len + 2; + ignored[header_params++] = hn; + } + + next_length: + + continue; } } @@ -869,7 +888,7 @@ ngx_http_scgi_create_request(ngx_http_re for (n = 0; n < header_params; n++) { if (&header[i] == ignored[n]) { - goto next; + goto next_value; } } @@ -893,12 +912,33 @@ ngx_http_scgi_create_request(ngx_http_re val = b->last; b->last = ngx_copy(val, header[i].value.data, header[i].value.len); + + if (header[i].next) { + + if (header[i].key.len == sizeof("Cookie") - 1 + && ngx_strncasecmp(header[i].key.data, (u_char *) "Cookie", + sizeof("Cookie") - 1) + == 0) + { + sep = ';'; + + } else { + sep = ','; + } + + for (hn = header[i].next; hn; hn = hn->next) { + *b->last++ = sep; + *b->last++ = ' '; + b->last = ngx_copy(b->last, hn->value.data, hn->value.len); + } + } + *b->last++ = (u_char) 0; ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "scgi param: \"%s: %s\"", key, val); - next: + next_value: continue; } From mdounin at mdounin.ru Wed Apr 20 22:18:42 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:42 +0300 Subject: [PATCH 02 of 20] FastCGI: combining headers with identical names (ticket #1724) In-Reply-To: References: Message-ID: <61b29233a55216c6fa72.1650493122@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492316 -10800 # Thu Apr 21 01:05:16 2022 +0300 # Node ID 61b29233a55216c6fa72e23b93a4a28d76a9fb94 # Parent e70fb0fdfbc0fb7b7e9f493cc2eb65de617b115a FastCGI: combining headers with identical names (ticket #1724). FastCGI responder is expected to receive CGI/1.1 environment variables in the parameters (see section "6.2 Responder" of the FastCGI specification). Obviously enough, there cannot be multiple environment variables with the same name. Further, CGI specification (RFC 3875, section "4.1.18. Protocol-Specific Meta-Variables") explicitly requires to combine headers: "If multiple header fields with the same field-name are received then the server MUST rewrite them as a single value having the same semantics". diff --git a/src/core/ngx_hash.h b/src/core/ngx_hash.h --- a/src/core/ngx_hash.h +++ b/src/core/ngx_hash.h @@ -89,12 +89,15 @@ typedef struct { } ngx_hash_keys_arrays_t; -typedef struct { +typedef struct ngx_table_elt_s ngx_table_elt_t; + +struct ngx_table_elt_s { ngx_uint_t hash; ngx_str_t key; ngx_str_t value; u_char *lowcase_key; -} ngx_table_elt_t; + ngx_table_elt_t *next; +}; void *ngx_hash_find(ngx_hash_t *hash, ngx_uint_t key, u_char *name, size_t len); diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c --- a/src/http/modules/ngx_http_fastcgi_module.c +++ b/src/http/modules/ngx_http_fastcgi_module.c @@ -835,14 +835,14 @@ static ngx_int_t ngx_http_fastcgi_create_request(ngx_http_request_t *r) { off_t file_pos; - u_char ch, *pos, *lowcase_key; + u_char ch, sep, *pos, *lowcase_key; size_t size, len, key_len, val_len, padding, allocated; ngx_uint_t i, n, next, hash, skip_empty, header_params; ngx_buf_t *b; ngx_chain_t *cl, *body; ngx_list_part_t *part; - ngx_table_elt_t *header, **ignored; + ngx_table_elt_t *header, *hn, **ignored; ngx_http_upstream_t *u; ngx_http_script_code_pt code; ngx_http_script_engine_t e, le; @@ -900,7 +900,11 @@ ngx_http_fastcgi_create_request(ngx_http allocated = 0; lowcase_key = NULL; - if (params->number) { + if (ngx_http_link_multi_headers(r) != NGX_OK) { + return NGX_ERROR; + } + + if (params->number || r->headers_in.multi) { n = 0; part = &r->headers_in.headers.part; @@ -930,6 +934,12 @@ ngx_http_fastcgi_create_request(ngx_http i = 0; } + for (n = 0; n < header_params; n++) { + if (&header[i] == ignored[n]) { + goto next_length; + } + } + if (params->number) { if (allocated < header[i].key.len) { allocated = header[i].key.len + 16; @@ -959,15 +969,23 @@ ngx_http_fastcgi_create_request(ngx_http ignored[header_params++] = &header[i]; continue; } - - n += sizeof("HTTP_") - 1; - - } else { - n = sizeof("HTTP_") - 1 + header[i].key.len; } - len += ((n > 127) ? 4 : 1) + ((header[i].value.len > 127) ? 4 : 1) - + n + header[i].value.len; + key_len = sizeof("HTTP_") - 1 + header[i].key.len; + + val_len = header[i].value.len; + + for (hn = header[i].next; hn; hn = hn->next) { + val_len += hn->value.len + 2; + ignored[header_params++] = hn; + } + + len += ((key_len > 127) ? 4 : 1) + key_len + + ((val_len > 127) ? 4 : 1) + val_len; + + next_length: + + continue; } } @@ -1109,7 +1127,7 @@ ngx_http_fastcgi_create_request(ngx_http for (n = 0; n < header_params; n++) { if (&header[i] == ignored[n]) { - goto next; + goto next_value; } } @@ -1125,6 +1143,11 @@ ngx_http_fastcgi_create_request(ngx_http } val_len = header[i].value.len; + + for (hn = header[i].next; hn; hn = hn->next) { + val_len += hn->value.len + 2; + } + if (val_len > 127) { *b->last++ = (u_char) (((val_len >> 24) & 0x7f) | 0x80); *b->last++ = (u_char) ((val_len >> 16) & 0xff); @@ -1150,13 +1173,34 @@ ngx_http_fastcgi_create_request(ngx_http *b->last++ = ch; } - b->last = ngx_copy(b->last, header[i].value.data, val_len); + b->last = ngx_copy(b->last, header[i].value.data, + header[i].value.len); + + if (header[i].next) { + + if (header[i].key.len == sizeof("Cookie") - 1 + && ngx_strncasecmp(header[i].key.data, (u_char *) "Cookie", + sizeof("Cookie") - 1) + == 0) + { + sep = ';'; + + } else { + sep = ','; + } + + for (hn = header[i].next; hn; hn = hn->next) { + *b->last++ = sep; + *b->last++ = ' '; + b->last = ngx_copy(b->last, hn->value.data, hn->value.len); + } + } ngx_log_debug4(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "fastcgi param: \"%*s: %*s\"", key_len, b->last - (key_len + val_len), val_len, b->last - val_len); - next: + next_value: continue; } diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -2802,6 +2802,78 @@ ngx_http_get_forwarded_addr_internal(ngx } +ngx_int_t +ngx_http_link_multi_headers(ngx_http_request_t *r) +{ + ngx_uint_t i, j; + ngx_list_part_t *part, *ppart; + ngx_table_elt_t *header, *pheader, **ph; + + if (r->headers_in.multi_linked) { + return NGX_OK; + } + + part = &r->headers_in.headers.part; + header = part->elts; + + for (i = 0; /* void */; i++) { + + if (i >= part->nelts) { + if (part->next == NULL) { + break; + } + + part = part->next; + header = part->elts; + i = 0; + } + + header[i].next = NULL; + + /* + * search for previous headers with the same name; + * if there are any, link to them + */ + + ppart = &r->headers_in.headers.part; + pheader = part->elts; + + for (j = 0; /* void */; j++) { + + if (j >= ppart->nelts) { + if (ppart->next == NULL) { + break; + } + + ppart = ppart->next; + pheader = ppart->elts; + i = 0; + } + + if (part == ppart && i == j) { + break; + } + + if (header[i].key.len == pheader[j].key.len + && ngx_strncasecmp(header[i].key.data, pheader[j].key.data, + header[i].key.len) + == 0) + { + ph = &pheader[j].next; + while (*ph) { ph = &(*ph)->next; } + *ph = &header[i]; + + r->headers_in.multi = 1; + + break; + } + } + } + + return NGX_OK; +} + + static char * ngx_http_core_server(ngx_conf_t *cf, ngx_command_t *cmd, void *dummy) { diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -532,6 +532,8 @@ ngx_int_t ngx_http_get_forwarded_addr(ng ngx_array_t *headers, ngx_str_t *value, ngx_array_t *proxies, int recursive); +ngx_int_t ngx_http_link_multi_headers(ngx_http_request_t *r); + extern ngx_module_t ngx_http_core_module; diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -242,6 +242,8 @@ typedef struct { unsigned connection_type:2; unsigned chunked:1; + unsigned multi:1; + unsigned multi_linked:1; unsigned msie:1; unsigned msie6:1; unsigned opera:1; From mdounin at mdounin.ru Wed Apr 20 22:18:44 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:44 +0300 Subject: [PATCH 04 of 20] Uwsgi: combining headers with identical names (ticket #1724) In-Reply-To: References: Message-ID: <3618f35ac3c8833cfd4a.1650493124@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492320 -10800 # Thu Apr 21 01:05:20 2022 +0300 # Node ID 3618f35ac3c8833cfd4a4c80295ab94eb924a9a7 # Parent 99c3ac43b556830c90ed4e8cf179d429a4830112 Uwsgi: combining headers with identical names (ticket #1724). The uwsgi specification states that "The uwsgi block vars represent a dictionary/hash". This implies that no duplicate headers are expected. Further, provided headers are expected to follow CGI specification, which also requires to combine headers (RFC 3875, section "4.1.18. Protocol-Specific Meta-Variables"): "If multiple header fields with the same field-name are received then the server MUST rewrite them as a single value having the same semantics". diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -845,13 +845,13 @@ ngx_http_uwsgi_create_key(ngx_http_reque static ngx_int_t ngx_http_uwsgi_create_request(ngx_http_request_t *r) { - u_char ch, *lowcase_key; + u_char ch, sep, *lowcase_key; size_t key_len, val_len, len, allocated; ngx_uint_t i, n, hash, skip_empty, header_params; ngx_buf_t *b; ngx_chain_t *cl, *body; ngx_list_part_t *part; - ngx_table_elt_t *header, **ignored; + ngx_table_elt_t *header, *hn, **ignored; ngx_http_uwsgi_params_t *params; ngx_http_script_code_pt code; ngx_http_script_engine_t e, le; @@ -905,7 +905,11 @@ ngx_http_uwsgi_create_request(ngx_http_r allocated = 0; lowcase_key = NULL; - if (params->number) { + if (ngx_http_link_multi_headers(r) != NGX_OK) { + return NGX_ERROR; + } + + if (params->number || r->headers_in.multi) { n = 0; part = &r->headers_in.headers.part; @@ -935,6 +939,12 @@ ngx_http_uwsgi_create_request(ngx_http_r i = 0; } + for (n = 0; n < header_params; n++) { + if (&header[i] == ignored[n]) { + goto next_length; + } + } + if (params->number) { if (allocated < header[i].key.len) { allocated = header[i].key.len + 16; @@ -968,6 +978,15 @@ ngx_http_uwsgi_create_request(ngx_http_r len += 2 + sizeof("HTTP_") - 1 + header[i].key.len + 2 + header[i].value.len; + + for (hn = header[i].next; hn; hn = hn->next) { + len += hn->value.len + 2; + ignored[header_params++] = hn; + } + + next_length: + + continue; } } @@ -1086,7 +1105,7 @@ ngx_http_uwsgi_create_request(ngx_http_r for (n = 0; n < header_params; n++) { if (&header[i] == ignored[n]) { - goto next; + goto next_value; } } @@ -1109,15 +1128,41 @@ ngx_http_uwsgi_create_request(ngx_http_r } val_len = header[i].value.len; + + for (hn = header[i].next; hn; hn = hn->next) { + val_len += hn->value.len + 2; + } + *b->last++ = (u_char) (val_len & 0xff); *b->last++ = (u_char) ((val_len >> 8) & 0xff); - b->last = ngx_copy(b->last, header[i].value.data, val_len); + b->last = ngx_copy(b->last, header[i].value.data, + header[i].value.len); + + if (header[i].next) { + + if (header[i].key.len == sizeof("Cookie") - 1 + && ngx_strncasecmp(header[i].key.data, (u_char *) "Cookie", + sizeof("Cookie") - 1) + == 0) + { + sep = ';'; + + } else { + sep = ','; + } + + for (hn = header[i].next; hn; hn = hn->next) { + *b->last++ = sep; + *b->last++ = ' '; + b->last = ngx_copy(b->last, hn->value.data, hn->value.len); + } + } ngx_log_debug4(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "uwsgi param: \"%*s: %*s\"", key_len, b->last - (key_len + 2 + val_len), val_len, b->last - val_len); - next: + next_value: continue; } From mdounin at mdounin.ru Wed Apr 20 22:18:48 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:48 +0300 Subject: [PATCH 08 of 20] Perl: all known input headers are handled identically In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1650492326 -10800 # Thu Apr 21 01:05:26 2022 +0300 # Node ID bf163552d6a9919f3c614aeb975202eda35c07c0 # Parent 238d1f5f438735432822689605fa37b1b0e01517 Perl: all known input headers are handled identically. As all known input headers are now linked lists, these are now handled identically. In particular, this makes it possible to access properly combined values of headers not specifically handled previously, such as "Via" or "Connection". diff --git a/src/http/modules/perl/nginx.xs b/src/http/modules/perl/nginx.xs --- a/src/http/modules/perl/nginx.xs +++ b/src/http/modules/perl/nginx.xs @@ -304,28 +304,10 @@ header_in(r, key) if (hh->offset == offsetof(ngx_http_headers_in_t, cookie)) { sep = ';'; - goto multi; - } -#if (NGX_HTTP_X_FORWARDED_FOR) - if (hh->offset == offsetof(ngx_http_headers_in_t, x_forwarded_for)) { + + } else { sep = ','; - goto multi; } -#endif - - ph = (ngx_table_elt_t **) ((char *) &r->headers_in + hh->offset); - - if (*ph) { - ngx_http_perl_set_targ((*ph)->value.data, (*ph)->value.len); - - goto done; - } - - XSRETURN_UNDEF; - - multi: - - /* Cookie, X-Forwarded-For */ ph = (ngx_table_elt_t **) ((char *) &r->headers_in + hh->offset); From mdounin at mdounin.ru Wed Apr 20 22:18:47 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:47 +0300 Subject: [PATCH 07 of 20] All non-unique input headers are now linked lists In-Reply-To: References: Message-ID: <238d1f5f438735432822.1650493127@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492324 -10800 # Thu Apr 21 01:05:24 2022 +0300 # Node ID 238d1f5f438735432822689605fa37b1b0e01517 # Parent 50fe52f516ff9c148aa9e7dfcc1c31cc6a4929ae All non-unique input headers are now linked lists. The ngx_http_process_multi_header_lines() function is removed, as it is exactly equivalent to ngx_http_process_header_line(). Similarly, ngx_http_variable_header() is used instead of ngx_http_variable_headers(). diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -22,8 +22,6 @@ static ngx_int_t ngx_http_process_header ngx_table_elt_t *h, ngx_uint_t offset); static ngx_int_t ngx_http_process_unique_header_line(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset); -static ngx_int_t ngx_http_process_multi_header_lines(ngx_http_request_t *r, - ngx_table_elt_t *h, ngx_uint_t offset); static ngx_int_t ngx_http_process_host(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset); static ngx_int_t ngx_http_process_connection(ngx_http_request_t *r, @@ -164,7 +162,7 @@ ngx_http_header_t ngx_http_headers_in[] #if (NGX_HTTP_X_FORWARDED_FOR) { ngx_string("X-Forwarded-For"), offsetof(ngx_http_headers_in_t, x_forwarded_for), - ngx_http_process_multi_header_lines }, + ngx_http_process_header_line }, #endif #if (NGX_HTTP_REALIP) @@ -197,7 +195,7 @@ ngx_http_header_t ngx_http_headers_in[] #endif { ngx_string("Cookie"), offsetof(ngx_http_headers_in_t, cookie), - ngx_http_process_multi_header_lines }, + ngx_http_process_header_line }, { ngx_null_string, 0, NULL } }; @@ -1742,10 +1740,10 @@ ngx_http_process_header_line(ngx_http_re ph = (ngx_table_elt_t **) ((char *) &r->headers_in + offset); - if (*ph == NULL) { - *ph = h; - h->next = NULL; - } + while (*ph) { ph = &(*ph)->next; } + + *ph = h; + h->next = NULL; return NGX_OK; } @@ -1851,13 +1849,10 @@ ngx_http_process_user_agent(ngx_http_req { u_char *user_agent, *msie; - if (r->headers_in.user_agent) { - return NGX_OK; + if (ngx_http_process_header_line(r, h, offset) != NGX_OK) { + return NGX_ERROR; } - r->headers_in.user_agent = h; - h->next = NULL; - /* check some widespread browsers while the header is in CPU cache */ user_agent = h->value.data; @@ -1919,23 +1914,6 @@ ngx_http_process_user_agent(ngx_http_req } -static ngx_int_t -ngx_http_process_multi_header_lines(ngx_http_request_t *r, ngx_table_elt_t *h, - ngx_uint_t offset) -{ - ngx_table_elt_t **ph; - - ph = (ngx_table_elt_t **) ((char *) &r->headers_in + offset); - - while (*ph) { ph = &(*ph)->next; } - - *ph = h; - h->next = NULL; - - return NGX_OK; -} - - ngx_int_t ngx_http_process_request_header(ngx_http_request_t *r) { diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c --- a/src/http/ngx_http_variables.c +++ b/src/http/ngx_http_variables.c @@ -27,8 +27,6 @@ static ngx_int_t ngx_http_variable_heade static ngx_int_t ngx_http_variable_cookies(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); -static ngx_int_t ngx_http_variable_headers(ngx_http_request_t *r, - ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_headers_internal(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data, u_char sep); @@ -178,7 +176,7 @@ static ngx_http_variable_t ngx_http_cor #endif #if (NGX_HTTP_X_FORWARDED_FOR) - { ngx_string("http_x_forwarded_for"), NULL, ngx_http_variable_headers, + { ngx_string("http_x_forwarded_for"), NULL, ngx_http_variable_header, offsetof(ngx_http_request_t, headers_in.x_forwarded_for), 0, 0 }, #endif @@ -327,10 +325,10 @@ static ngx_http_variable_t ngx_http_cor { ngx_string("sent_http_transfer_encoding"), NULL, ngx_http_variable_sent_transfer_encoding, 0, 0, 0 }, - { ngx_string("sent_http_cache_control"), NULL, ngx_http_variable_headers, + { ngx_string("sent_http_cache_control"), NULL, ngx_http_variable_header, offsetof(ngx_http_request_t, headers_out.cache_control), 0, 0 }, - { ngx_string("sent_http_link"), NULL, ngx_http_variable_headers, + { ngx_string("sent_http_link"), NULL, ngx_http_variable_header, offsetof(ngx_http_request_t, headers_out.link), 0, 0 }, { ngx_string("limit_rate"), ngx_http_variable_set_limit_rate, @@ -807,22 +805,7 @@ static ngx_int_t ngx_http_variable_header(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { - ngx_table_elt_t *h; - - h = *(ngx_table_elt_t **) ((char *) r + data); - - if (h) { - v->len = h->value.len; - v->valid = 1; - v->no_cacheable = 0; - v->not_found = 0; - v->data = h->value.data; - - } else { - v->not_found = 1; - } - - return NGX_OK; + return ngx_http_variable_headers_internal(r, v, data, ','); } @@ -835,14 +818,6 @@ ngx_http_variable_cookies(ngx_http_reque static ngx_int_t -ngx_http_variable_headers(ngx_http_request_t *r, - ngx_http_variable_value_t *v, uintptr_t data) -{ - return ngx_http_variable_headers_internal(r, v, data, ','); -} - - -static ngx_int_t ngx_http_variable_headers_internal(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data, u_char sep) { From mdounin at mdounin.ru Wed Apr 20 22:18:50 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:50 +0300 Subject: [PATCH 10 of 20] Upstream: style In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1650492329 -10800 # Thu Apr 21 01:05:29 2022 +0300 # Node ID ee1a8a4aa2c262d25a4aa871cda4f6c4515fc85c # Parent 70dc7705dd6953a58a62a9e7588f71151562fc40 Upstream: style. diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -4691,8 +4691,8 @@ static ngx_int_t ngx_http_upstream_process_cache_control(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { - ngx_table_elt_t **ph; - ngx_http_upstream_t *u; + ngx_table_elt_t **ph; + ngx_http_upstream_t *u; u = r->upstream; ph = &u->headers_in.cache_control; @@ -4980,7 +4980,11 @@ static ngx_int_t ngx_http_upstream_process_charset(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { - if (r->upstream->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_CHARSET) { + ngx_http_upstream_t *u; + + u = r->upstream; + + if (u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_CHARSET) { return NGX_OK; } @@ -4994,13 +4998,16 @@ static ngx_int_t ngx_http_upstream_process_connection(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { - r->upstream->headers_in.connection = h; + ngx_http_upstream_t *u; + + u = r->upstream; + u->headers_in.connection = h; if (ngx_strlcasestrn(h->value.data, h->value.data + h->value.len, (u_char *) "close", 5 - 1) != NULL) { - r->upstream->headers_in.connection_close = 1; + u->headers_in.connection_close = 1; } return NGX_OK; @@ -5011,13 +5018,16 @@ static ngx_int_t ngx_http_upstream_process_transfer_encoding(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { - r->upstream->headers_in.transfer_encoding = h; + ngx_http_upstream_t *u; + + u = r->upstream; + u->headers_in.transfer_encoding = h; if (ngx_strlcasestrn(h->value.data, h->value.data + h->value.len, (u_char *) "chunked", 7 - 1) != NULL) { - r->upstream->headers_in.chunked = 1; + u->headers_in.chunked = 1; } return NGX_OK; From mdounin at mdounin.ru Wed Apr 20 22:18:49 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:49 +0300 Subject: [PATCH 09 of 20] Perl: combining unknown headers during $r->header_in() lookup In-Reply-To: References: Message-ID: <70dc7705dd6953a58a62.1650493129@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492327 -10800 # Thu Apr 21 01:05:27 2022 +0300 # Node ID 70dc7705dd6953a58a62a9e7588f71151562fc40 # Parent bf163552d6a9919f3c614aeb975202eda35c07c0 Perl: combining unknown headers during $r->header_in() lookup. diff --git a/src/http/modules/perl/nginx.xs b/src/http/modules/perl/nginx.xs --- a/src/http/modules/perl/nginx.xs +++ b/src/http/modules/perl/nginx.xs @@ -272,7 +272,7 @@ header_in(r, key) ngx_uint_t i, n, hash; ngx_array_t *a; ngx_list_part_t *part; - ngx_table_elt_t *h, **ph; + ngx_table_elt_t *h, *header, **ph; ngx_http_header_t *hh; ngx_http_core_main_conf_t *cmcf; @@ -311,47 +311,14 @@ header_in(r, key) ph = (ngx_table_elt_t **) ((char *) &r->headers_in + hh->offset); - if (*ph == NULL) { - XSRETURN_UNDEF; - } - - if ((*ph)->next == NULL) { - ngx_http_perl_set_targ((*ph)->value.data, (*ph)->value.len); - - goto done; - } - - size = - (ssize_t) (sizeof("; ") - 1); - - for (h = *ph; h; h = h->next) { - size += h->value.len + sizeof("; ") - 1; - } - - value = ngx_pnalloc(r->pool, size); - if (value == NULL) { - ctx->error = 1; - croak("ngx_pnalloc() failed"); - } - - p = value; - - for (h = *ph; h; h = h->next) { - p = ngx_copy(p, h->value.data, h->value.len); - - if (h->next == NULL) { - break; - } - - *p++ = sep; *p++ = ' '; - } - - ngx_http_perl_set_targ(value, size); - - goto done; + goto found; } /* iterate over all headers */ + sep = ','; + ph = &header; + part = &r->headers_in.headers.part; h = part->elts; @@ -373,12 +340,49 @@ header_in(r, key) continue; } - ngx_http_perl_set_targ(h[i].value.data, h[i].value.len); + *ph = &h[i]; + ph = &h[i].next; + } + + *ph = NULL; + ph = &header; + found: + + if (*ph == NULL) { + XSRETURN_UNDEF; + } + + if ((*ph)->next == NULL) { + ngx_http_perl_set_targ((*ph)->value.data, (*ph)->value.len); goto done; } - XSRETURN_UNDEF; + size = - (ssize_t) (sizeof("; ") - 1); + + for (h = *ph; h; h = h->next) { + size += h->value.len + sizeof("; ") - 1; + } + + value = ngx_pnalloc(r->pool, size); + if (value == NULL) { + ctx->error = 1; + croak("ngx_pnalloc() failed"); + } + + p = value; + + for (h = *ph; h; h = h->next) { + p = ngx_copy(p, h->value.data, h->value.len); + + if (h->next == NULL) { + break; + } + + *p++ = sep; *p++ = ' '; + } + + ngx_http_perl_set_targ(value, size); done: From mdounin at mdounin.ru Wed Apr 20 22:18:52 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:52 +0300 Subject: [PATCH 12 of 20] Upstream: simplified Accept-Ranges handling In-Reply-To: References: Message-ID: <1231bff732696207842f.1650493132@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492331 -10800 # Thu Apr 21 01:05:31 2022 +0300 # Node ID 1231bff732696207842fa66b71ca63b951ac5c06 # Parent 6441069e16a0c4755c662bc07bdcb0960c9ac04a Upstream: simplified Accept-Ranges handling. The u->headers_in.accept_ranges field is not used anywhere and hence removed. diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -259,8 +259,7 @@ static ngx_http_upstream_header_t ngx_h offsetof(ngx_http_headers_out_t, expires), 1 }, { ngx_string("Accept-Ranges"), - ngx_http_upstream_process_header_line, - offsetof(ngx_http_upstream_headers_in_t, accept_ranges), + ngx_http_upstream_ignore_header_line, 0, ngx_http_upstream_copy_allow_ranges, offsetof(ngx_http_headers_out_t, accept_ranges), 1 }, diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -280,7 +280,6 @@ typedef struct { ngx_table_elt_t *last_modified; ngx_table_elt_t *location; - ngx_table_elt_t *accept_ranges; ngx_table_elt_t *www_authenticate; ngx_table_elt_t *transfer_encoding; ngx_table_elt_t *vary; From mdounin at mdounin.ru Wed Apr 20 22:18:51 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:51 +0300 Subject: [PATCH 11 of 20] Upstream: simplified Content-Encoding handling In-Reply-To: References: Message-ID: <6441069e16a0c4755c66.1650493131@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492330 -10800 # Thu Apr 21 01:05:30 2022 +0300 # Node ID 6441069e16a0c4755c662bc07bdcb0960c9ac04a # Parent ee1a8a4aa2c262d25a4aa871cda4f6c4515fc85c Upstream: simplified Content-Encoding handling. Since introduction of offset handling in ngx_http_upstream_copy_header_line() in revision 573:58475592100c, the ngx_http_upstream_copy_content_encoding() function is no longer needed, as its behaviour is exactly equivalent to ngx_http_upstream_copy_header_line() with appropriate offset. As such, the ngx_http_upstream_copy_content_encoding() function was removed. Further, the u->headers_in.content_encoding field is not used anywhere, so it was removed as well. Further, Content-Encoding handling no longer depends on NGX_HTTP_GZIP, as it can be used even without any gzip handling compiled in (for example, in the charset filter). diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -147,11 +147,6 @@ static ngx_int_t ngx_http_upstream_rewri static ngx_int_t ngx_http_upstream_copy_allow_ranges(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset); -#if (NGX_HTTP_GZIP) -static ngx_int_t ngx_http_upstream_copy_content_encoding(ngx_http_request_t *r, - ngx_table_elt_t *h, ngx_uint_t offset); -#endif - static ngx_int_t ngx_http_upstream_add_variables(ngx_conf_t *cf); static ngx_int_t ngx_http_upstream_addr_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); @@ -316,12 +311,10 @@ static ngx_http_upstream_header_t ngx_h ngx_http_upstream_process_transfer_encoding, 0, ngx_http_upstream_ignore_header_line, 0, 0 }, -#if (NGX_HTTP_GZIP) { ngx_string("Content-Encoding"), - ngx_http_upstream_process_header_line, - offsetof(ngx_http_upstream_headers_in_t, content_encoding), - ngx_http_upstream_copy_content_encoding, 0, 0 }, -#endif + ngx_http_upstream_ignore_header_line, 0, + ngx_http_upstream_copy_header_line, + offsetof(ngx_http_headers_out_t, content_encoding), 0 }, { ngx_null_string, NULL, 0, NULL, 0, 0 } }; @@ -5349,29 +5342,6 @@ ngx_http_upstream_copy_allow_ranges(ngx_ } -#if (NGX_HTTP_GZIP) - -static ngx_int_t -ngx_http_upstream_copy_content_encoding(ngx_http_request_t *r, - ngx_table_elt_t *h, ngx_uint_t offset) -{ - ngx_table_elt_t *ho; - - ho = ngx_list_push(&r->headers_out.headers); - if (ho == NULL) { - return NGX_ERROR; - } - - *ho = *h; - - r->headers_out.content_encoding = ho; - - return NGX_OK; -} - -#endif - - static ngx_int_t ngx_http_upstream_add_variables(ngx_conf_t *cf) { diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -285,10 +285,6 @@ typedef struct { ngx_table_elt_t *transfer_encoding; ngx_table_elt_t *vary; -#if (NGX_HTTP_GZIP) - ngx_table_elt_t *content_encoding; -#endif - ngx_table_elt_t *cache_control; ngx_table_elt_t *set_cookie; From mdounin at mdounin.ru Wed Apr 20 22:18:45 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:45 +0300 Subject: [PATCH 05 of 20] Combining unknown headers during variables lookup (ticket #1316) In-Reply-To: References: Message-ID: <2ea48b5e4643a818cd81.1650493125@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492321 -10800 # Thu Apr 21 01:05:21 2022 +0300 # Node ID 2ea48b5e4643a818cd81179f040f0b36be9050d6 # Parent 3618f35ac3c8833cfd4a4c80295ab94eb924a9a7 Combining unknown headers during variables lookup (ticket #1316). Previously, $http_*, $sent_http_*, $upstream_http_*, and $upstream_trailer_* variables returned only the first header (with a few specially handled exceptions: $http_cookie, $http_x_forwarded_for, $sent_http_cache_control, $sent_http_link). With this change, all headers are returned, combined together. For example, $http_foo variable will be "a, b" if there are "Foo: a" and "Foo: b" headers in the request. Note that $upstream_http_set_cookie will also return all "Set-Cookie" headers (ticket #1843), though this might not be what one want, since the "Set-Cookie" header does not follow the list syntax (see RFC 7230, section 3.2.2). diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -5703,7 +5703,7 @@ ngx_http_upstream_header_variable(ngx_ht return NGX_OK; } - return ngx_http_variable_unknown_header(v, (ngx_str_t *) data, + return ngx_http_variable_unknown_header(r, v, (ngx_str_t *) data, &r->upstream->headers_in.headers.part, sizeof("upstream_http_") - 1); } @@ -5718,7 +5718,7 @@ ngx_http_upstream_trailer_variable(ngx_h return NGX_OK; } - return ngx_http_variable_unknown_header(v, (ngx_str_t *) data, + return ngx_http_variable_unknown_header(r, v, (ngx_str_t *) data, &r->upstream->headers_in.trailers.part, sizeof("upstream_trailer_") - 1); } diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c --- a/src/http/ngx_http_variables.c +++ b/src/http/ngx_http_variables.c @@ -919,7 +919,7 @@ static ngx_int_t ngx_http_variable_unknown_header_in(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { - return ngx_http_variable_unknown_header(v, (ngx_str_t *) data, + return ngx_http_variable_unknown_header(r, v, (ngx_str_t *) data, &r->headers_in.headers.part, sizeof("http_") - 1); } @@ -929,7 +929,7 @@ static ngx_int_t ngx_http_variable_unknown_header_out(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { - return ngx_http_variable_unknown_header(v, (ngx_str_t *) data, + return ngx_http_variable_unknown_header(r, v, (ngx_str_t *) data, &r->headers_out.headers.part, sizeof("sent_http_") - 1); } @@ -939,19 +939,26 @@ static ngx_int_t ngx_http_variable_unknown_trailer_out(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { - return ngx_http_variable_unknown_header(v, (ngx_str_t *) data, + return ngx_http_variable_unknown_header(r, v, (ngx_str_t *) data, &r->headers_out.trailers.part, sizeof("sent_trailer_") - 1); } ngx_int_t -ngx_http_variable_unknown_header(ngx_http_variable_value_t *v, ngx_str_t *var, +ngx_http_variable_unknown_header(ngx_http_request_t *r, + ngx_http_variable_value_t *v, ngx_str_t *var, ngx_list_part_t *part, size_t prefix) { - u_char ch; + u_char *p, ch; + size_t len; ngx_uint_t i, n; - ngx_table_elt_t *header; + ngx_table_elt_t *header, *h, **ph; + + ph = &h; +#if (NGX_SUPPRESS_WARN) + len = 0; +#endif header = part->elts; @@ -971,7 +978,11 @@ ngx_http_variable_unknown_header(ngx_htt continue; } - for (n = 0; n + prefix < var->len && n < header[i].key.len; n++) { + if (header[i].key.len != var->len - prefix) { + continue; + } + + for (n = 0; n < var->len - prefix; n++) { ch = header[i].key.data[n]; if (ch >= 'A' && ch <= 'Z') { @@ -986,18 +997,59 @@ ngx_http_variable_unknown_header(ngx_htt } } - if (n + prefix == var->len && n == header[i].key.len) { - v->len = header[i].value.len; - v->valid = 1; - v->no_cacheable = 0; - v->not_found = 0; - v->data = header[i].value.data; - - return NGX_OK; + if (n != var->len - prefix) { + continue; } + + len += header[i].value.len + 2; + + *ph = &header[i]; + ph = &header[i].next; } - v->not_found = 1; + *ph = NULL; + + if (h == NULL) { + v->not_found = 1; + return NGX_OK; + } + + len -= 2; + + if (h->next == NULL) { + + v->len = h->value.len; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = h->value.data; + + return NGX_OK; + } + + p = ngx_pnalloc(r->pool, len); + if (p == NULL) { + return NGX_ERROR; + } + + v->len = len; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = p; + + for ( ;; ) { + + p = ngx_copy(p, h->value.data, h->value.len); + + if (h->next == NULL) { + break; + } + + *p++ = ','; *p++ = ' '; + + h = h->next; + } return NGX_OK; } @@ -1879,7 +1931,7 @@ ngx_http_variable_sent_location(ngx_http ngx_str_set(&name, "sent_http_location"); - return ngx_http_variable_unknown_header(v, &name, + return ngx_http_variable_unknown_header(r, v, &name, &r->headers_out.headers.part, sizeof("sent_http_") - 1); } diff --git a/src/http/ngx_http_variables.h b/src/http/ngx_http_variables.h --- a/src/http/ngx_http_variables.h +++ b/src/http/ngx_http_variables.h @@ -57,8 +57,9 @@ ngx_http_variable_value_t *ngx_http_get_ ngx_http_variable_value_t *ngx_http_get_variable(ngx_http_request_t *r, ngx_str_t *name, ngx_uint_t key); -ngx_int_t ngx_http_variable_unknown_header(ngx_http_variable_value_t *v, - ngx_str_t *var, ngx_list_part_t *part, size_t prefix); +ngx_int_t ngx_http_variable_unknown_header(ngx_http_request_t *r, + ngx_http_variable_value_t *v, ngx_str_t *var, ngx_list_part_t *part, + size_t prefix); #if (NGX_PCRE) From mdounin at mdounin.ru Wed Apr 20 22:18:46 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:46 +0300 Subject: [PATCH 06 of 20] Reworked multi headers to use linked lists In-Reply-To: References: Message-ID: <50fe52f516ff9c148aa9.1650493126@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492323 -10800 # Thu Apr 21 01:05:23 2022 +0300 # Node ID 50fe52f516ff9c148aa9e7dfcc1c31cc6a4929ae # Parent 2ea48b5e4643a818cd81179f040f0b36be9050d6 Reworked multi headers to use linked lists. Multi headers are now using linked lists instead of arrays. Notably, the following fields were changed: r->headers_in.cookies (renamed to r->headers_in.cookie), r->headers_in.x_forwarded_for, r->headers_out.cache_control, r->headers_out.link, u->headers_in.cache_control u->headers_in.cookies (renamed to u->headers_in.set_cookie). The r->headers_in.cookies and u->headers_in.cookies fields were renamed to r->headers_in.cookie and u->headers_in.set_cookie to match header names. The ngx_http_parse_multi_header_lines() and ngx_http_parse_set_cookie_lines() functions were changed accordingly. With this change, multi headers are now essentially equivalent to normal headers, and following changes will further make them equivalent. diff --git a/src/http/modules/ngx_http_geo_module.c b/src/http/modules/ngx_http_geo_module.c --- a/src/http/modules/ngx_http_geo_module.c +++ b/src/http/modules/ngx_http_geo_module.c @@ -327,15 +327,15 @@ static ngx_int_t ngx_http_geo_addr(ngx_http_request_t *r, ngx_http_geo_ctx_t *ctx, ngx_addr_t *addr) { - ngx_array_t *xfwd; + ngx_table_elt_t *xfwd; if (ngx_http_geo_real_addr(r, ctx, addr) != NGX_OK) { return NGX_ERROR; } - xfwd = &r->headers_in.x_forwarded_for; + xfwd = r->headers_in.x_forwarded_for; - if (xfwd->nelts > 0 && ctx->proxies != NULL) { + if (xfwd != NULL && ctx->proxies != NULL) { (void) ngx_http_get_forwarded_addr(r, addr, xfwd, NULL, ctx->proxies, ctx->proxy_recursive); } diff --git a/src/http/modules/ngx_http_geoip_module.c b/src/http/modules/ngx_http_geoip_module.c --- a/src/http/modules/ngx_http_geoip_module.c +++ b/src/http/modules/ngx_http_geoip_module.c @@ -240,16 +240,16 @@ static u_long ngx_http_geoip_addr(ngx_http_request_t *r, ngx_http_geoip_conf_t *gcf) { ngx_addr_t addr; - ngx_array_t *xfwd; + ngx_table_elt_t *xfwd; struct sockaddr_in *sin; addr.sockaddr = r->connection->sockaddr; addr.socklen = r->connection->socklen; /* addr.name = r->connection->addr_text; */ - xfwd = &r->headers_in.x_forwarded_for; + xfwd = r->headers_in.x_forwarded_for; - if (xfwd->nelts > 0 && gcf->proxies != NULL) { + if (xfwd != NULL && gcf->proxies != NULL) { (void) ngx_http_get_forwarded_addr(r, &addr, xfwd, NULL, gcf->proxies, gcf->proxy_recursive); } @@ -292,7 +292,7 @@ static geoipv6_t ngx_http_geoip_addr_v6(ngx_http_request_t *r, ngx_http_geoip_conf_t *gcf) { ngx_addr_t addr; - ngx_array_t *xfwd; + ngx_table_elt_t *xfwd; in_addr_t addr4; struct in6_addr addr6; struct sockaddr_in *sin; @@ -302,9 +302,9 @@ ngx_http_geoip_addr_v6(ngx_http_request_ addr.socklen = r->connection->socklen; /* addr.name = r->connection->addr_text; */ - xfwd = &r->headers_in.x_forwarded_for; + xfwd = r->headers_in.x_forwarded_for; - if (xfwd->nelts > 0 && gcf->proxies != NULL) { + if (xfwd != NULL && gcf->proxies != NULL) { (void) ngx_http_get_forwarded_addr(r, &addr, xfwd, NULL, gcf->proxies, gcf->proxy_recursive); } diff --git a/src/http/modules/ngx_http_headers_filter_module.c b/src/http/modules/ngx_http_headers_filter_module.c --- a/src/http/modules/ngx_http_headers_filter_module.c +++ b/src/http/modules/ngx_http_headers_filter_module.c @@ -329,8 +329,7 @@ ngx_http_set_expires(ngx_http_request_t time_t now, expires_time, max_age; ngx_str_t value; ngx_int_t rc; - ngx_uint_t i; - ngx_table_elt_t *e, *cc, **ccp; + ngx_table_elt_t *e, *cc; ngx_http_expires_t expires; expires = conf->expires; @@ -371,38 +370,28 @@ ngx_http_set_expires(ngx_http_request_t len = sizeof("Mon, 28 Sep 1970 06:00:00 GMT"); e->value.len = len - 1; - ccp = r->headers_out.cache_control.elts; - - if (ccp == NULL) { + cc = r->headers_out.cache_control; - if (ngx_array_init(&r->headers_out.cache_control, r->pool, - 1, sizeof(ngx_table_elt_t *)) - != NGX_OK) - { - return NGX_ERROR; - } + if (cc == NULL) { cc = ngx_list_push(&r->headers_out.headers); if (cc == NULL) { return NGX_ERROR; } + r->headers_out.cache_control = cc; + cc->next = NULL; + cc->hash = 1; ngx_str_set(&cc->key, "Cache-Control"); - ccp = ngx_array_push(&r->headers_out.cache_control); - if (ccp == NULL) { - return NGX_ERROR; + } else { + for (cc = cc->next; cc; cc = cc->next) { + cc->hash = 0; } - *ccp = cc; - - } else { - for (i = 1; i < r->headers_out.cache_control.nelts; i++) { - ccp[i]->hash = 0; - } - - cc = ccp[0]; + cc = r->headers_out.cache_control; + cc->next = NULL; } if (expires == NGX_HTTP_EXPIRES_EPOCH) { @@ -564,22 +553,12 @@ static ngx_int_t ngx_http_add_multi_header_lines(ngx_http_request_t *r, ngx_http_header_val_t *hv, ngx_str_t *value) { - ngx_array_t *pa; ngx_table_elt_t *h, **ph; if (value->len == 0) { return NGX_OK; } - pa = (ngx_array_t *) ((char *) &r->headers_out + hv->offset); - - if (pa->elts == NULL) { - if (ngx_array_init(pa, r->pool, 1, sizeof(ngx_table_elt_t *)) != NGX_OK) - { - return NGX_ERROR; - } - } - h = ngx_list_push(&r->headers_out.headers); if (h == NULL) { return NGX_ERROR; @@ -589,12 +568,12 @@ ngx_http_add_multi_header_lines(ngx_http h->key = hv->key; h->value = *value; - ph = ngx_array_push(pa); - if (ph == NULL) { - return NGX_ERROR; - } + ph = (ngx_table_elt_t **) ((char *) &r->headers_out + hv->offset); + + while (*ph) { ph = &(*ph)->next; } *ph = h; + h->next = NULL; return NGX_OK; } diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -2559,22 +2559,20 @@ static ngx_int_t ngx_http_proxy_add_x_forwarded_for_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { - size_t len; - u_char *p; - ngx_uint_t i, n; - ngx_table_elt_t **h; + size_t len; + u_char *p; + ngx_table_elt_t *h, *xfwd; v->valid = 1; v->no_cacheable = 0; v->not_found = 0; - n = r->headers_in.x_forwarded_for.nelts; - h = r->headers_in.x_forwarded_for.elts; + xfwd = r->headers_in.x_forwarded_for; len = 0; - for (i = 0; i < n; i++) { - len += h[i]->value.len + sizeof(", ") - 1; + for (h = xfwd; h; h = h->next) { + len += h->value.len + sizeof(", ") - 1; } if (len == 0) { @@ -2593,8 +2591,8 @@ ngx_http_proxy_add_x_forwarded_for_varia v->len = len; v->data = p; - for (i = 0; i < n; i++) { - p = ngx_copy(p, h[i]->value.data, h[i]->value.len); + for (h = xfwd; h; h = h->next) { + p = ngx_copy(p, h->value.data, h->value.len); *p++ = ','; *p++ = ' '; } diff --git a/src/http/modules/ngx_http_realip_module.c b/src/http/modules/ngx_http_realip_module.c --- a/src/http/modules/ngx_http_realip_module.c +++ b/src/http/modules/ngx_http_realip_module.c @@ -134,9 +134,8 @@ ngx_http_realip_handler(ngx_http_request ngx_str_t *value; ngx_uint_t i, hash; ngx_addr_t addr; - ngx_array_t *xfwd; ngx_list_part_t *part; - ngx_table_elt_t *header; + ngx_table_elt_t *header, *xfwd; ngx_connection_t *c; ngx_http_realip_ctx_t *ctx; ngx_http_realip_loc_conf_t *rlcf; @@ -168,9 +167,9 @@ ngx_http_realip_handler(ngx_http_request case NGX_HTTP_REALIP_XFWD: - xfwd = &r->headers_in.x_forwarded_for; + xfwd = r->headers_in.x_forwarded_for; - if (xfwd->elts == NULL) { + if (xfwd == NULL) { return NGX_DECLINED; } diff --git a/src/http/modules/ngx_http_userid_filter_module.c b/src/http/modules/ngx_http_userid_filter_module.c --- a/src/http/modules/ngx_http_userid_filter_module.c +++ b/src/http/modules/ngx_http_userid_filter_module.c @@ -319,10 +319,9 @@ ngx_http_userid_set_variable(ngx_http_re static ngx_http_userid_ctx_t * ngx_http_userid_get_uid(ngx_http_request_t *r, ngx_http_userid_conf_t *conf) { - ngx_int_t n; - ngx_str_t src, dst; - ngx_table_elt_t **cookies; - ngx_http_userid_ctx_t *ctx; + ngx_str_t src, dst; + ngx_table_elt_t *cookie; + ngx_http_userid_ctx_t *ctx; ctx = ngx_http_get_module_ctx(r, ngx_http_userid_filter_module); @@ -339,9 +338,9 @@ ngx_http_userid_get_uid(ngx_http_request ngx_http_set_ctx(r, ctx, ngx_http_userid_filter_module); } - n = ngx_http_parse_multi_header_lines(&r->headers_in.cookies, &conf->name, - &ctx->cookie); - if (n == NGX_DECLINED) { + cookie = ngx_http_parse_multi_header_lines(r, r->headers_in.cookie, + &conf->name, &ctx->cookie); + if (cookie == NULL) { return ctx; } @@ -349,10 +348,9 @@ ngx_http_userid_get_uid(ngx_http_request "uid cookie: \"%V\"", &ctx->cookie); if (ctx->cookie.len < 22) { - cookies = r->headers_in.cookies.elts; ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "client sent too short userid cookie \"%V\"", - &cookies[n]->value); + &cookie->value); return ctx; } @@ -370,10 +368,9 @@ ngx_http_userid_get_uid(ngx_http_request dst.data = (u_char *) ctx->uid_got; if (ngx_decode_base64(&dst, &src) == NGX_ERROR) { - cookies = r->headers_in.cookies.elts; ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "client sent invalid userid cookie \"%V\"", - &cookies[n]->value); + &cookie->value); return ctx; } diff --git a/src/http/modules/perl/nginx.xs b/src/http/modules/perl/nginx.xs --- a/src/http/modules/perl/nginx.xs +++ b/src/http/modules/perl/nginx.xs @@ -302,7 +302,7 @@ header_in(r, key) if (hh) { - if (hh->offset == offsetof(ngx_http_headers_in_t, cookies)) { + if (hh->offset == offsetof(ngx_http_headers_in_t, cookie)) { sep = ';'; goto multi; } @@ -327,17 +327,13 @@ header_in(r, key) /* Cookie, X-Forwarded-For */ - a = (ngx_array_t *) ((char *) &r->headers_in + hh->offset); + ph = (ngx_table_elt_t **) ((char *) &r->headers_in + hh->offset); - n = a->nelts; - - if (n == 0) { + if (*ph == NULL) { XSRETURN_UNDEF; } - ph = a->elts; - - if (n == 1) { + if ((*ph)->next == NULL) { ngx_http_perl_set_targ((*ph)->value.data, (*ph)->value.len); goto done; @@ -345,8 +341,8 @@ header_in(r, key) size = - (ssize_t) (sizeof("; ") - 1); - for (i = 0; i < n; i++) { - size += ph[i]->value.len + sizeof("; ") - 1; + for (h = *ph; h; h = h->next) { + size += h->value.len + sizeof("; ") - 1; } value = ngx_pnalloc(r->pool, size); @@ -357,10 +353,10 @@ header_in(r, key) p = value; - for (i = 0; /* void */ ; i++) { - p = ngx_copy(p, ph[i]->value.data, ph[i]->value.len); + for (h = *ph; h; h = h->next) { + p = ngx_copy(p, h->value.data, h->value.len); - if (i == n - 1) { + if (h->next == NULL) { break; } diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h --- a/src/http/ngx_http.h +++ b/src/http/ngx_http.h @@ -103,10 +103,10 @@ ngx_int_t ngx_http_parse_unsafe_uri(ngx_ ngx_str_t *args, ngx_uint_t *flags); ngx_int_t ngx_http_parse_header_line(ngx_http_request_t *r, ngx_buf_t *b, ngx_uint_t allow_underscores); -ngx_int_t ngx_http_parse_multi_header_lines(ngx_array_t *headers, - ngx_str_t *name, ngx_str_t *value); -ngx_int_t ngx_http_parse_set_cookie_lines(ngx_array_t *headers, - ngx_str_t *name, ngx_str_t *value); +ngx_table_elt_t *ngx_http_parse_multi_header_lines(ngx_http_request_t *r, + ngx_table_elt_t *headers, ngx_str_t *name, ngx_str_t *value); +ngx_table_elt_t *ngx_http_parse_set_cookie_lines(ngx_http_request_t *r, + ngx_table_elt_t *headers, ngx_str_t *name, ngx_str_t *value); ngx_int_t ngx_http_arg(ngx_http_request_t *r, u_char *name, size_t len, ngx_str_t *value); void ngx_http_split_args(ngx_http_request_t *r, ngx_str_t *uri, diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -2024,8 +2024,7 @@ ngx_http_gzip_ok(ngx_http_request_t *r) { time_t date, expires; ngx_uint_t p; - ngx_array_t *cc; - ngx_table_elt_t *e, *d, *ae; + ngx_table_elt_t *e, *d, *ae, *cc; ngx_http_core_loc_conf_t *clcf; r->gzip_tested = 1; @@ -2118,30 +2117,30 @@ ngx_http_gzip_ok(ngx_http_request_t *r) return NGX_DECLINED; } - cc = &r->headers_out.cache_control; - - if (cc->elts) { + cc = r->headers_out.cache_control; + + if (cc) { if ((p & NGX_HTTP_GZIP_PROXIED_NO_CACHE) - && ngx_http_parse_multi_header_lines(cc, &ngx_http_gzip_no_cache, + && ngx_http_parse_multi_header_lines(r, cc, &ngx_http_gzip_no_cache, NULL) - >= 0) + != NULL) { goto ok; } if ((p & NGX_HTTP_GZIP_PROXIED_NO_STORE) - && ngx_http_parse_multi_header_lines(cc, &ngx_http_gzip_no_store, + && ngx_http_parse_multi_header_lines(r, cc, &ngx_http_gzip_no_store, NULL) - >= 0) + != NULL) { goto ok; } if ((p & NGX_HTTP_GZIP_PROXIED_PRIVATE) - && ngx_http_parse_multi_header_lines(cc, &ngx_http_gzip_private, + && ngx_http_parse_multi_header_lines(r, cc, &ngx_http_gzip_private, NULL) - >= 0) + != NULL) { goto ok; } @@ -2712,12 +2711,12 @@ ngx_http_set_disable_symlinks(ngx_http_r ngx_int_t ngx_http_get_forwarded_addr(ngx_http_request_t *r, ngx_addr_t *addr, - ngx_array_t *headers, ngx_str_t *value, ngx_array_t *proxies, + ngx_table_elt_t *headers, ngx_str_t *value, ngx_array_t *proxies, int recursive) { - ngx_int_t rc; - ngx_uint_t i, found; - ngx_table_elt_t **h; + ngx_int_t rc; + ngx_uint_t found; + ngx_table_elt_t *h, *next; if (headers == NULL) { return ngx_http_get_forwarded_addr_internal(r, addr, value->data, @@ -2725,16 +2724,23 @@ ngx_http_get_forwarded_addr(ngx_http_req recursive); } - i = headers->nelts; - h = headers->elts; + /* revert headers order */ + + for (h = headers, headers = NULL; h; h = next) { + next = h->next; + h->next = headers; + headers = h; + } + + /* iterate over all headers in reverse order */ rc = NGX_DECLINED; found = 0; - while (i-- > 0) { - rc = ngx_http_get_forwarded_addr_internal(r, addr, h[i]->value.data, - h[i]->value.len, proxies, + for (h = headers; h; h = h->next) { + rc = ngx_http_get_forwarded_addr_internal(r, addr, h->value.data, + h->value.len, proxies, recursive); if (!recursive) { @@ -2753,6 +2759,14 @@ ngx_http_get_forwarded_addr(ngx_http_req found = 1; } + /* restore headers order */ + + for (h = headers, headers = NULL; h; h = next) { + next = h->next; + h->next = headers; + headers = h; + } + return rc; } diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -529,7 +529,7 @@ ngx_int_t ngx_http_set_disable_symlinks( ngx_http_core_loc_conf_t *clcf, ngx_str_t *path, ngx_open_file_info_t *of); ngx_int_t ngx_http_get_forwarded_addr(ngx_http_request_t *r, ngx_addr_t *addr, - ngx_array_t *headers, ngx_str_t *value, ngx_array_t *proxies, + ngx_table_elt_t *headers, ngx_str_t *value, ngx_array_t *proxies, int recursive); ngx_int_t ngx_http_link_multi_headers(ngx_http_request_t *r); diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c --- a/src/http/ngx_http_parse.c +++ b/src/http/ngx_http_parse.c @@ -1960,27 +1960,24 @@ unsafe: } -ngx_int_t -ngx_http_parse_multi_header_lines(ngx_array_t *headers, ngx_str_t *name, - ngx_str_t *value) +ngx_table_elt_t * +ngx_http_parse_multi_header_lines(ngx_http_request_t *r, + ngx_table_elt_t *headers, ngx_str_t *name, ngx_str_t *value) { - ngx_uint_t i; - u_char *start, *last, *end, ch; - ngx_table_elt_t **h; - - h = headers->elts; - - for (i = 0; i < headers->nelts; i++) { - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, headers->pool->log, 0, - "parse header: \"%V: %V\"", &h[i]->key, &h[i]->value); - - if (name->len > h[i]->value.len) { + u_char *start, *last, *end, ch; + ngx_table_elt_t *h; + + for (h = headers; h; h = h->next) { + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "parse header: \"%V: %V\"", &h->key, &h->value); + + if (name->len > h->value.len) { continue; } - start = h[i]->value.data; - end = h[i]->value.data + h[i]->value.len; + start = h->value.data; + end = h->value.data + h->value.len; while (start < end) { @@ -1994,7 +1991,7 @@ ngx_http_parse_multi_header_lines(ngx_ar if (value == NULL) { if (start == end || *start == ',') { - return i; + return h; } goto skip; @@ -2014,7 +2011,7 @@ ngx_http_parse_multi_header_lines(ngx_ar value->len = last - start; value->data = start; - return i; + return h; skip: @@ -2029,31 +2026,28 @@ ngx_http_parse_multi_header_lines(ngx_ar } } - return NGX_DECLINED; + return NULL; } -ngx_int_t -ngx_http_parse_set_cookie_lines(ngx_array_t *headers, ngx_str_t *name, - ngx_str_t *value) +ngx_table_elt_t * +ngx_http_parse_set_cookie_lines(ngx_http_request_t *r, + ngx_table_elt_t *headers, ngx_str_t *name, ngx_str_t *value) { - ngx_uint_t i; - u_char *start, *last, *end; - ngx_table_elt_t **h; - - h = headers->elts; - - for (i = 0; i < headers->nelts; i++) { - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, headers->pool->log, 0, - "parse header: \"%V: %V\"", &h[i]->key, &h[i]->value); - - if (name->len >= h[i]->value.len) { + u_char *start, *last, *end; + ngx_table_elt_t *h; + + for (h = headers; h; h = h->next) { + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "parse header: \"%V: %V\"", &h->key, &h->value); + + if (name->len >= h->value.len) { continue; } - start = h[i]->value.data; - end = h[i]->value.data + h[i]->value.len; + start = h->value.data; + end = h->value.data + h->value.len; if (ngx_strncasecmp(start, name->data, name->len) != 0) { continue; @@ -2077,10 +2071,10 @@ ngx_http_parse_set_cookie_lines(ngx_arra value->len = last - start; value->data = start; - return i; + return h; } - return NGX_DECLINED; + return NULL; } diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -196,7 +196,7 @@ ngx_http_header_t ngx_http_headers_in[] ngx_http_process_header_line }, #endif - { ngx_string("Cookie"), offsetof(ngx_http_headers_in_t, cookies), + { ngx_string("Cookie"), offsetof(ngx_http_headers_in_t, cookie), ngx_http_process_multi_header_lines }, { ngx_null_string, 0, NULL } @@ -1744,6 +1744,7 @@ ngx_http_process_header_line(ngx_http_re if (*ph == NULL) { *ph = h; + h->next = NULL; } return NGX_OK; @@ -1760,6 +1761,7 @@ ngx_http_process_unique_header_line(ngx_ if (*ph == NULL) { *ph = h; + h->next = NULL; return NGX_OK; } @@ -1792,6 +1794,7 @@ ngx_http_process_host(ngx_http_request_t } r->headers_in.host = h; + h->next = NULL; host = h->value; @@ -1853,6 +1856,7 @@ ngx_http_process_user_agent(ngx_http_req } r->headers_in.user_agent = h; + h->next = NULL; /* check some widespread browsers while the header is in CPU cache */ @@ -1919,27 +1923,15 @@ static ngx_int_t ngx_http_process_multi_header_lines(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { - ngx_array_t *headers; ngx_table_elt_t **ph; - headers = (ngx_array_t *) ((char *) &r->headers_in + offset); - - if (headers->elts == NULL) { - if (ngx_array_init(headers, r->pool, 1, sizeof(ngx_table_elt_t *)) - != NGX_OK) - { - ngx_http_close_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); - return NGX_ERROR; - } - } - - ph = ngx_array_push(headers); - if (ph == NULL) { - ngx_http_close_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); - return NGX_ERROR; - } + ph = (ngx_table_elt_t **) ((char *) &r->headers_in + offset); + + while (*ph) { ph = &(*ph)->next; } *ph = h; + h->next = NULL; + return NGX_OK; } diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -212,7 +212,7 @@ typedef struct { ngx_table_elt_t *keep_alive; #if (NGX_HTTP_X_FORWARDED_FOR) - ngx_array_t x_forwarded_for; + ngx_table_elt_t *x_forwarded_for; #endif #if (NGX_HTTP_REALIP) @@ -231,11 +231,11 @@ typedef struct { ngx_table_elt_t *date; #endif + ngx_table_elt_t *cookie; + ngx_str_t user; ngx_str_t passwd; - ngx_array_t cookies; - ngx_str_t server; off_t content_length_n; time_t keep_alive_n; @@ -274,6 +274,9 @@ typedef struct { ngx_table_elt_t *expires; ngx_table_elt_t *etag; + ngx_table_elt_t *cache_control; + ngx_table_elt_t *link; + ngx_str_t *override_charset; size_t content_type_len; @@ -282,9 +285,6 @@ typedef struct { u_char *content_type_lowcase; ngx_uint_t content_type_hash; - ngx_array_t cache_control; - ngx_array_t link; - off_t content_length_n; off_t content_offset; time_t date_time; diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -246,7 +246,7 @@ static ngx_http_upstream_header_t ngx_h { ngx_string("Set-Cookie"), ngx_http_upstream_process_set_cookie, - offsetof(ngx_http_upstream_headers_in_t, cookies), + offsetof(ngx_http_upstream_headers_in_t, set_cookie), ngx_http_upstream_rewrite_set_cookie, 0, 1 }, { ngx_string("Content-Disposition"), @@ -4666,26 +4666,16 @@ static ngx_int_t ngx_http_upstream_process_set_cookie(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { - ngx_array_t *pa; ngx_table_elt_t **ph; ngx_http_upstream_t *u; u = r->upstream; - pa = &u->headers_in.cookies; - - if (pa->elts == NULL) { - if (ngx_array_init(pa, r->pool, 1, sizeof(ngx_table_elt_t *)) != NGX_OK) - { - return NGX_ERROR; - } - } - - ph = ngx_array_push(pa); - if (ph == NULL) { - return NGX_ERROR; - } + ph = &u->headers_in.set_cookie; + + while (*ph) { ph = &(*ph)->next; } *ph = h; + h->next = NULL; #if (NGX_HTTP_CACHE) if (!(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_SET_COOKIE)) { @@ -4701,26 +4691,16 @@ static ngx_int_t ngx_http_upstream_process_cache_control(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { - ngx_array_t *pa; ngx_table_elt_t **ph; ngx_http_upstream_t *u; u = r->upstream; - pa = &u->headers_in.cache_control; - - if (pa->elts == NULL) { - if (ngx_array_init(pa, r->pool, 2, sizeof(ngx_table_elt_t *)) != NGX_OK) - { - return NGX_ERROR; - } - } - - ph = ngx_array_push(pa); - if (ph == NULL) { - return NGX_ERROR; - } + ph = &u->headers_in.cache_control; + + while (*ph) { ph = &(*ph)->next; } *ph = h; + h->next = NULL; #if (NGX_HTTP_CACHE) { @@ -5103,18 +5083,8 @@ static ngx_int_t ngx_http_upstream_copy_multi_header_lines(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { - ngx_array_t *pa; ngx_table_elt_t *ho, **ph; - pa = (ngx_array_t *) ((char *) &r->headers_out + offset); - - if (pa->elts == NULL) { - if (ngx_array_init(pa, r->pool, 2, sizeof(ngx_table_elt_t *)) != NGX_OK) - { - return NGX_ERROR; - } - } - ho = ngx_list_push(&r->headers_out.headers); if (ho == NULL) { return NGX_ERROR; @@ -5122,12 +5092,12 @@ ngx_http_upstream_copy_multi_header_line *ho = *h; - ph = ngx_array_push(pa); - if (ph == NULL) { - return NGX_ERROR; - } + ph = (ngx_table_elt_t **) ((char *) &r->headers_out + offset); + + while (*ph) { ph = &(*ph)->next; } *ph = ho; + ho->next = NULL; return NGX_OK; } @@ -5740,9 +5710,9 @@ ngx_http_upstream_cookie_variable(ngx_ht s.len = name->len - (sizeof("upstream_cookie_") - 1); s.data = name->data + sizeof("upstream_cookie_") - 1; - if (ngx_http_parse_set_cookie_lines(&r->upstream->headers_in.cookies, + if (ngx_http_parse_set_cookie_lines(r, r->upstream->headers_in.set_cookie, &s, &cookie) - == NGX_DECLINED) + == NULL) { v->not_found = 1; return NGX_OK; diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -289,8 +289,8 @@ typedef struct { ngx_table_elt_t *content_encoding; #endif - ngx_array_t cache_control; - ngx_array_t cookies; + ngx_table_elt_t *cache_control; + ngx_table_elt_t *set_cookie; off_t content_length_n; time_t last_modified_time; diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c --- a/src/http/ngx_http_variables.c +++ b/src/http/ngx_http_variables.c @@ -183,7 +183,7 @@ static ngx_http_variable_t ngx_http_cor #endif { ngx_string("http_cookie"), NULL, ngx_http_variable_cookies, - offsetof(ngx_http_request_t, headers_in.cookies), 0, 0 }, + offsetof(ngx_http_request_t, headers_in.cookie), 0, 0 }, { ngx_string("content_length"), NULL, ngx_http_variable_content_length, 0, 0, 0 }, @@ -846,26 +846,21 @@ static ngx_int_t ngx_http_variable_headers_internal(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data, u_char sep) { - size_t len; - u_char *p, *end; - ngx_uint_t i, n; - ngx_array_t *a; - ngx_table_elt_t **h; - - a = (ngx_array_t *) ((char *) r + data); - - n = a->nelts; - h = a->elts; + size_t len; + u_char *p; + ngx_table_elt_t *h, *th; + + h = *(ngx_table_elt_t **) ((char *) r + data); len = 0; - for (i = 0; i < n; i++) { - - if (h[i]->hash == 0) { + for (th = h; th; th = th->next) { + + if (th->hash == 0) { continue; } - len += h[i]->value.len + 2; + len += th->value.len + 2; } if (len == 0) { @@ -879,9 +874,9 @@ ngx_http_variable_headers_internal(ngx_h v->no_cacheable = 0; v->not_found = 0; - if (n == 1) { - v->len = (*h)->value.len; - v->data = (*h)->value.data; + if (h->next == NULL) { + v->len = h->value.len; + v->data = h->value.data; return NGX_OK; } @@ -894,17 +889,15 @@ ngx_http_variable_headers_internal(ngx_h v->len = len; v->data = p; - end = p + len; - - for (i = 0; /* void */ ; i++) { - - if (h[i]->hash == 0) { + for (th = h; th; th = th->next) { + + if (th->hash == 0) { continue; } - p = ngx_copy(p, h[i]->value.data, h[i]->value.len); - - if (p == end) { + p = ngx_copy(p, th->value.data, th->value.len); + + if (th->next == NULL) { break; } @@ -1102,8 +1095,8 @@ ngx_http_variable_cookie(ngx_http_reques s.len = name->len - (sizeof("cookie_") - 1); s.data = name->data + sizeof("cookie_") - 1; - if (ngx_http_parse_multi_header_lines(&r->headers_in.cookies, &s, &cookie) - == NGX_DECLINED) + if (ngx_http_parse_multi_header_lines(r, r->headers_in.cookie, &s, &cookie) + == NULL) { v->not_found = 1; return NGX_OK; diff --git a/src/http/v2/ngx_http_v2_filter_module.c b/src/http/v2/ngx_http_v2_filter_module.c --- a/src/http/v2/ngx_http_v2_filter_module.c +++ b/src/http/v2/ngx_http_v2_filter_module.c @@ -674,14 +674,14 @@ ngx_http_v2_header_filter(ngx_http_reque static ngx_int_t ngx_http_v2_push_resources(ngx_http_request_t *r) { - u_char *start, *end, *last; - ngx_int_t rc; - ngx_str_t path; - ngx_uint_t i, push; - ngx_table_elt_t **h; - ngx_http_v2_loc_conf_t *h2lcf; - ngx_http_complex_value_t *pushes; - ngx_str_t binary[NGX_HTTP_V2_PUSH_HEADERS]; + u_char *start, *end, *last; + ngx_int_t rc; + ngx_str_t path; + ngx_uint_t i, push; + ngx_table_elt_t *h; + ngx_http_v2_loc_conf_t *h2lcf; + ngx_http_complex_value_t *pushes; + ngx_str_t binary[NGX_HTTP_V2_PUSH_HEADERS]; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http2 push resources"); @@ -725,15 +725,13 @@ ngx_http_v2_push_resources(ngx_http_requ return NGX_OK; } - h = r->headers_out.link.elts; - - for (i = 0; i < r->headers_out.link.nelts; i++) { + for (h = r->headers_out.link; h; h = h->next) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http2 parse link: \"%V\"", &h[i]->value); - - start = h[i]->value.data; - end = h[i]->value.data + h[i]->value.len; + "http2 parse link: \"%V\"", &h->value); + + start = h->value.data; + end = h->value.data + h->value.len; next_link: From mdounin at mdounin.ru Wed Apr 20 22:18:54 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:54 +0300 Subject: [PATCH 14 of 20] Upstream: all known headers in u->headers_in are linked lists now In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1650492335 -10800 # Thu Apr 21 01:05:35 2022 +0300 # Node ID b110c54778e8f6af3ea402c0838a4f289dcd813e # Parent 618f9f39e9ad53fd84e94d0c2a0f1b7f82bfcca5 Upstream: all known headers in u->headers_in are linked lists now. diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -1965,6 +1965,7 @@ ngx_http_proxy_process_header(ngx_http_r ngx_str_set(&h->key, "Server"); ngx_str_null(&h->value); h->lowcase_key = (u_char *) "server"; + h->next = NULL; } if (r->upstream->headers_in.date == NULL) { @@ -1978,6 +1979,7 @@ ngx_http_proxy_process_header(ngx_http_r ngx_str_set(&h->key, "Date"); ngx_str_null(&h->value); h->lowcase_key = (u_char *) "date"; + h->next = NULL; } /* clear content length if response is chunked */ diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -4610,6 +4610,7 @@ ngx_http_upstream_process_header_line(ng if (*ph == NULL) { *ph = h; + h->next = NULL; } return NGX_OK; @@ -4632,6 +4633,7 @@ ngx_http_upstream_process_content_length u = r->upstream; + h->next = NULL; u->headers_in.content_length = h; u->headers_in.content_length_n = ngx_atoof(h->value.data, h->value.len); @@ -4647,6 +4649,7 @@ ngx_http_upstream_process_last_modified( u = r->upstream; + h->next = NULL; u->headers_in.last_modified = h; u->headers_in.last_modified_time = ngx_parse_http_time(h->value.data, h->value.len); @@ -4816,6 +4819,7 @@ ngx_http_upstream_process_expires(ngx_ht u = r->upstream; u->headers_in.expires = h; + h->next = NULL; #if (NGX_HTTP_CACHE) { @@ -4856,6 +4860,7 @@ ngx_http_upstream_process_accel_expires( u = r->upstream; u->headers_in.x_accel_expires = h; + h->next = NULL; #if (NGX_HTTP_CACHE) { @@ -4915,6 +4920,7 @@ ngx_http_upstream_process_limit_rate(ngx u = r->upstream; u->headers_in.x_accel_limit_rate = h; + h->next = NULL; if (u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_LIMIT_RATE) { return NGX_OK; @@ -4995,6 +5001,7 @@ ngx_http_upstream_process_connection(ngx u = r->upstream; u->headers_in.connection = h; + h->next = NULL; if (ngx_strlcasestrn(h->value.data, h->value.data + h->value.len, (u_char *) "close", 5 - 1) @@ -5015,6 +5022,7 @@ ngx_http_upstream_process_transfer_encod u = r->upstream; u->headers_in.transfer_encoding = h; + h->next = NULL; if (ngx_strlcasestrn(h->value.data, h->value.data + h->value.len, (u_char *) "chunked", 7 - 1) @@ -5035,6 +5043,7 @@ ngx_http_upstream_process_vary(ngx_http_ u = r->upstream; u->headers_in.vary = h; + h->next = NULL; #if (NGX_HTTP_CACHE) From mdounin at mdounin.ru Wed Apr 20 22:18:53 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:53 +0300 Subject: [PATCH 13 of 20] All known output headers can be linked lists now In-Reply-To: References: Message-ID: <618f9f39e9ad53fd84e9.1650493133@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492333 -10800 # Thu Apr 21 01:05:33 2022 +0300 # Node ID 618f9f39e9ad53fd84e94d0c2a0f1b7f82bfcca5 # Parent 1231bff732696207842fa66b71ca63b951ac5c06 All known output headers can be linked lists now. The h->next pointer properly provided as NULL in all cases where known output headers are added. Note that there are 3rd party modules which might not do this, and it might be risky to rely on this for arbitrary headers. diff --git a/src/http/modules/ngx_http_auth_basic_module.c b/src/http/modules/ngx_http_auth_basic_module.c --- a/src/http/modules/ngx_http_auth_basic_module.c +++ b/src/http/modules/ngx_http_auth_basic_module.c @@ -339,6 +339,7 @@ ngx_http_auth_basic_set_realm(ngx_http_r *p = '"'; r->headers_out.www_authenticate->hash = 1; + r->headers_out.www_authenticate->next = NULL; ngx_str_set(&r->headers_out.www_authenticate->key, "WWW-Authenticate"); r->headers_out.www_authenticate->value.data = basic; r->headers_out.www_authenticate->value.len = len; diff --git a/src/http/modules/ngx_http_auth_request_module.c b/src/http/modules/ngx_http_auth_request_module.c --- a/src/http/modules/ngx_http_auth_request_module.c +++ b/src/http/modules/ngx_http_auth_request_module.c @@ -154,6 +154,7 @@ ngx_http_auth_request_handler(ngx_http_r } *ho = *h; + ho->next = NULL; r->headers_out.www_authenticate = ho; } diff --git a/src/http/modules/ngx_http_dav_module.c b/src/http/modules/ngx_http_dav_module.c --- a/src/http/modules/ngx_http_dav_module.c +++ b/src/http/modules/ngx_http_dav_module.c @@ -1082,6 +1082,7 @@ ngx_http_dav_location(ngx_http_request_t } r->headers_out.location->hash = 1; + r->headers_out.location->next = NULL; ngx_str_set(&r->headers_out.location->key, "Location"); escape = 2 * ngx_escape_uri(NULL, r->uri.data, r->uri.len, NGX_ESCAPE_URI); diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c --- a/src/http/modules/ngx_http_gzip_filter_module.c +++ b/src/http/modules/ngx_http_gzip_filter_module.c @@ -280,6 +280,7 @@ ngx_http_gzip_header_filter(ngx_http_req } h->hash = 1; + h->next = NULL; ngx_str_set(&h->key, "Content-Encoding"); ngx_str_set(&h->value, "gzip"); r->headers_out.content_encoding = h; diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c --- a/src/http/modules/ngx_http_gzip_static_module.c +++ b/src/http/modules/ngx_http_gzip_static_module.c @@ -242,6 +242,7 @@ ngx_http_gzip_static_handler(ngx_http_re } h->hash = 1; + h->next = NULL; ngx_str_set(&h->key, "Content-Encoding"); ngx_str_set(&h->value, "gzip"); r->headers_out.content_encoding = h; diff --git a/src/http/modules/ngx_http_headers_filter_module.c b/src/http/modules/ngx_http_headers_filter_module.c --- a/src/http/modules/ngx_http_headers_filter_module.c +++ b/src/http/modules/ngx_http_headers_filter_module.c @@ -362,6 +362,7 @@ ngx_http_set_expires(ngx_http_request_t } r->headers_out.expires = e; + e->next = NULL; e->hash = 1; ngx_str_set(&e->key, "Expires"); @@ -621,6 +622,7 @@ ngx_http_set_response_header(ngx_http_re } *old = h; + h->next = NULL; } h->hash = 1; diff --git a/src/http/modules/ngx_http_memcached_module.c b/src/http/modules/ngx_http_memcached_module.c --- a/src/http/modules/ngx_http_memcached_module.c +++ b/src/http/modules/ngx_http_memcached_module.c @@ -401,6 +401,7 @@ found: } h->hash = 1; + h->next = NULL; ngx_str_set(&h->key, "Content-Encoding"); ngx_str_set(&h->value, "gzip"); r->headers_out.content_encoding = h; diff --git a/src/http/modules/ngx_http_range_filter_module.c b/src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c +++ b/src/http/modules/ngx_http_range_filter_module.c @@ -258,6 +258,7 @@ next_filter: } r->headers_out.accept_ranges->hash = 1; + r->headers_out.accept_ranges->next = NULL; ngx_str_set(&r->headers_out.accept_ranges->key, "Accept-Ranges"); ngx_str_set(&r->headers_out.accept_ranges->value, "bytes"); @@ -427,6 +428,7 @@ ngx_http_range_singlepart_header(ngx_htt r->headers_out.content_range = content_range; content_range->hash = 1; + content_range->next = NULL; ngx_str_set(&content_range->key, "Content-Range"); content_range->value.data = ngx_pnalloc(r->pool, @@ -599,6 +601,7 @@ ngx_http_range_not_satisfiable(ngx_http_ r->headers_out.content_range = content_range; content_range->hash = 1; + content_range->next = NULL; ngx_str_set(&content_range->key, "Content-Range"); content_range->value.data = ngx_pnalloc(r->pool, diff --git a/src/http/modules/ngx_http_static_module.c b/src/http/modules/ngx_http_static_module.c --- a/src/http/modules/ngx_http_static_module.c +++ b/src/http/modules/ngx_http_static_module.c @@ -195,6 +195,7 @@ ngx_http_static_handler(ngx_http_request } r->headers_out.location->hash = 1; + r->headers_out.location->next = NULL; ngx_str_set(&r->headers_out.location->key, "Location"); r->headers_out.location->value.len = len; r->headers_out.location->value.data = location; diff --git a/src/http/modules/perl/nginx.xs b/src/http/modules/perl/nginx.xs --- a/src/http/modules/perl/nginx.xs +++ b/src/http/modules/perl/nginx.xs @@ -573,6 +573,7 @@ header_out(r, key, value) } header->hash = 1; + header->next = NULL; if (ngx_http_perl_sv2str(aTHX_ r, &header->key, key) != NGX_OK) { header->hash = 0; diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -1007,6 +1007,7 @@ ngx_http_core_find_config_phase(ngx_http } r->headers_out.location->hash = 1; + r->headers_out.location->next = NULL; ngx_str_set(&r->headers_out.location->key, "Location"); if (r->args.len == 0) { @@ -1687,6 +1688,7 @@ ngx_http_set_etag(ngx_http_request_t *r) } etag->hash = 1; + etag->next = NULL; ngx_str_set(&etag->key, "ETag"); etag->value.data = ngx_pnalloc(r->pool, NGX_OFF_T_LEN + NGX_TIME_T_LEN + 3); @@ -1781,6 +1783,7 @@ ngx_http_send_response(ngx_http_request_ } r->headers_out.location->hash = 1; + r->headers_out.location->next = NULL; ngx_str_set(&r->headers_out.location->key, "Location"); r->headers_out.location->value = val; diff --git a/src/http/ngx_http_script.c b/src/http/ngx_http_script.c --- a/src/http/ngx_http_script.c +++ b/src/http/ngx_http_script.c @@ -1243,6 +1243,7 @@ ngx_http_script_regex_end_code(ngx_http_ } r->headers_out.location->hash = 1; + r->headers_out.location->next = NULL; ngx_str_set(&r->headers_out.location->key, "Location"); r->headers_out.location->value = e->buf; diff --git a/src/http/ngx_http_special_response.c b/src/http/ngx_http_special_response.c --- a/src/http/ngx_http_special_response.c +++ b/src/http/ngx_http_special_response.c @@ -649,6 +649,7 @@ ngx_http_send_error_page(ngx_http_reques } location->hash = 1; + location->next = NULL; ngx_str_set(&location->key, "Location"); location->value = uri; diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -2681,6 +2681,7 @@ ngx_http_upstream_intercept_errors(ngx_h } *h = *u->headers_in.www_authenticate; + h->next = NULL; r->headers_out.www_authenticate = h; } @@ -5075,6 +5076,7 @@ ngx_http_upstream_copy_header_line(ngx_h if (offset) { ph = (ngx_table_elt_t **) ((char *) &r->headers_out + offset); *ph = ho; + ho->next = NULL; } return NGX_OK; @@ -5169,6 +5171,7 @@ ngx_http_upstream_copy_last_modified(ngx } *ho = *h; + ho->next = NULL; r->headers_out.last_modified = ho; r->headers_out.last_modified_time = @@ -5191,6 +5194,7 @@ ngx_http_upstream_rewrite_location(ngx_h } *ho = *h; + ho->next = NULL; if (r->upstream->rewrite_redirect) { rc = r->upstream->rewrite_redirect(r, ho, 0); @@ -5236,6 +5240,7 @@ ngx_http_upstream_rewrite_refresh(ngx_ht } *ho = *h; + ho->next = NULL; if (r->upstream->rewrite_redirect) { @@ -5281,6 +5286,7 @@ ngx_http_upstream_rewrite_set_cookie(ngx } *ho = *h; + ho->next = NULL; if (r->upstream->rewrite_cookie) { rc = r->upstream->rewrite_cookie(r, ho); @@ -5334,6 +5340,7 @@ ngx_http_upstream_copy_allow_ranges(ngx_ } *ho = *h; + ho->next = NULL; r->headers_out.accept_ranges = ho; From mdounin at mdounin.ru Wed Apr 20 22:18:55 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:55 +0300 Subject: [PATCH 15 of 20] Upstream: header handlers can now return parsing errors In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1650492336 -10800 # Thu Apr 21 01:05:36 2022 +0300 # Node ID ab424b5e32405aeec54ccdfe38e9408209209e0a # Parent b110c54778e8f6af3ea402c0838a4f289dcd813e Upstream: header handlers can now return parsing errors. With this change, duplicate Content-Length and Transfer-Encoding headers are now rejected. Further, responses with invalid Content-Length or Transfer-Encoding headers are now rejected, as well as responses with both Content-Length and Transfer-Encoding. diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c --- a/src/http/modules/ngx_http_fastcgi_module.c +++ b/src/http/modules/ngx_http_fastcgi_module.c @@ -2007,8 +2007,12 @@ ngx_http_fastcgi_process_header(ngx_http hh = ngx_hash_find(&umcf->headers_in_hash, h->hash, h->lowcase_key, h->key.len); - if (hh && hh->handler(r, h, hh->offset) != NGX_OK) { - return NGX_ERROR; + if (hh) { + rc = hh->handler(r, h, hh->offset); + + if (rc != NGX_OK) { + return rc; + } } ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -1891,8 +1891,12 @@ ngx_http_grpc_process_header(ngx_http_re hh = ngx_hash_find(&umcf->headers_in_hash, h->hash, h->lowcase_key, h->key.len); - if (hh && hh->handler(r, h, hh->offset) != NGX_OK) { - return NGX_ERROR; + if (hh) { + rc = hh->handler(r, h, hh->offset); + + if (rc != NGX_OK) { + return rc; + } } continue; diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -1930,8 +1930,12 @@ ngx_http_proxy_process_header(ngx_http_r hh = ngx_hash_find(&umcf->headers_in_hash, h->hash, h->lowcase_key, h->key.len); - if (hh && hh->handler(r, h, hh->offset) != NGX_OK) { - return NGX_ERROR; + if (hh) { + rc = hh->handler(r, h, hh->offset); + + if (rc != NGX_OK) { + return rc; + } } ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c --- a/src/http/modules/ngx_http_scgi_module.c +++ b/src/http/modules/ngx_http_scgi_module.c @@ -1114,8 +1114,12 @@ ngx_http_scgi_process_header(ngx_http_re hh = ngx_hash_find(&umcf->headers_in_hash, h->hash, h->lowcase_key, h->key.len); - if (hh && hh->handler(r, h, hh->offset) != NGX_OK) { - return NGX_ERROR; + if (hh) { + rc = hh->handler(r, h, hh->offset); + + if (rc != NGX_OK) { + return rc; + } } ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -1340,8 +1340,12 @@ ngx_http_uwsgi_process_header(ngx_http_r hh = ngx_hash_find(&umcf->headers_in_hash, h->hash, h->lowcase_key, h->key.len); - if (hh && hh->handler(r, h, hh->offset) != NGX_OK) { - return NGX_ERROR; + if (hh) { + rc = hh->handler(r, h, hh->offset); + + if (rc != NGX_OK) { + return rc; + } } ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -4633,10 +4633,34 @@ ngx_http_upstream_process_content_length u = r->upstream; + if (u->headers_in.content_length) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "upstream sent duplicate header line: \"%V: %V\", " + "previous value: \"%V: %V\"", + &h->key, &h->value, + &u->headers_in.content_length->key, + &u->headers_in.content_length->value); + return NGX_HTTP_UPSTREAM_INVALID_HEADER; + } + + if (u->headers_in.transfer_encoding) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "upstream sent \"Content-Length\" and " + "\"Transfer-Encoding\" headers at the same time"); + return NGX_HTTP_UPSTREAM_INVALID_HEADER; + } + h->next = NULL; u->headers_in.content_length = h; u->headers_in.content_length_n = ngx_atoof(h->value.data, h->value.len); + if (u->headers_in.content_length_n == NGX_ERROR) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "upstream sent invalid \"Content-Length\" header: " + "\"%V: %V\"", &h->key, &h->value); + return NGX_HTTP_UPSTREAM_INVALID_HEADER; + } + return NGX_OK; } @@ -5021,14 +5045,37 @@ ngx_http_upstream_process_transfer_encod ngx_http_upstream_t *u; u = r->upstream; + + if (u->headers_in.transfer_encoding) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "upstream sent duplicate header line: \"%V: %V\", " + "previous value: \"%V: %V\"", + &h->key, &h->value, + &u->headers_in.transfer_encoding->key, + &u->headers_in.transfer_encoding->value); + return NGX_HTTP_UPSTREAM_INVALID_HEADER; + } + + if (u->headers_in.content_length) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "upstream sent \"Content-Length\" and " + "\"Transfer-Encoding\" headers at the same time"); + return NGX_HTTP_UPSTREAM_INVALID_HEADER; + } + u->headers_in.transfer_encoding = h; h->next = NULL; - if (ngx_strlcasestrn(h->value.data, h->value.data + h->value.len, - (u_char *) "chunked", 7 - 1) - != NULL) + if (h->value.len == 7 + && ngx_strncasecmp(h->value.data, (u_char *) "chunked", 7) == 0) { u->headers_in.chunked = 1; + + } else { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "upstream sent unknown \"Transfer-Encoding\": \"%V\"", + &h->value); + return NGX_HTTP_UPSTREAM_INVALID_HEADER; } return NGX_OK; From mdounin at mdounin.ru Wed Apr 20 22:18:57 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:57 +0300 Subject: [PATCH 17 of 20] Upstream: handling of multiple Vary headers (ticket #1423) In-Reply-To: References: Message-ID: <2027b85971d4b8a7e33c.1650493137@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492340 -10800 # Thu Apr 21 01:05:40 2022 +0300 # Node ID 2027b85971d4b8a7e33c018548468057cb57eaf7 # Parent f460a2f9f88d264ef6c8588eb37bcb85c48010db Upstream: handling of multiple Vary headers (ticket #1423). Previously, only the last header value was used when caching. diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -5175,6 +5175,9 @@ static ngx_int_t ngx_http_upstream_process_vary(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { + u_char *p; + size_t len; + ngx_str_t vary; ngx_table_elt_t **ph; ngx_http_upstream_t *u; @@ -5192,17 +5195,47 @@ ngx_http_upstream_process_vary(ngx_http_ return NGX_OK; } - if (r->cache == NULL) { + if (r->cache == NULL || !u->cacheable) { + return NGX_OK; + } + + if (h->value.len == 1 && h->value.data[0] == '*') { + u->cacheable = 0; return NGX_OK; } - if (h->value.len > NGX_HTTP_CACHE_VARY_LEN - || (h->value.len == 1 && h->value.data[0] == '*')) - { + if (u->headers_in.vary->next) { + + len = 0; + + for (h = u->headers_in.vary; h; h = h->next) { + len += h->value.len + 2; + } + + len -= 2; + + p = ngx_pnalloc(r->pool, len); + if (p == NULL) { + return NGX_ERROR; + } + + vary.len = len; + vary.data = p; + + for (h = u->headers_in.vary; h; h = h->next) { + p = ngx_copy(p, h->value.data, h->value.len); + *p++ = ','; *p++ = ' '; + } + + } else { + vary = h->value; + } + + if (vary.len > NGX_HTTP_CACHE_VARY_LEN) { u->cacheable = 0; } - r->cache->vary = h->value; + r->cache->vary = vary; #endif From mdounin at mdounin.ru Wed Apr 20 22:18:58 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:58 +0300 Subject: [PATCH 18 of 20] Upstream: multiple WWW-Authenticate headers (ticket #485) In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1650492341 -10800 # Thu Apr 21 01:05:41 2022 +0300 # Node ID fa9751ffe7723a11159c158078e454671e81cb87 # Parent 2027b85971d4b8a7e33c018548468057cb57eaf7 Upstream: multiple WWW-Authenticate headers (ticket #485). When using proxy_intercept_errors and an error page for error 401 (Unauthorized), multiple WWW-Authenticate headers from the upstream server response are now properly copied to the response. diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -2647,7 +2647,7 @@ ngx_http_upstream_intercept_errors(ngx_h { ngx_int_t status; ngx_uint_t i; - ngx_table_elt_t *h; + ngx_table_elt_t *h, *ho, **ph; ngx_http_err_page_t *err_page; ngx_http_core_loc_conf_t *clcf; @@ -2676,18 +2676,26 @@ ngx_http_upstream_intercept_errors(ngx_h if (status == NGX_HTTP_UNAUTHORIZED && u->headers_in.www_authenticate) { - h = ngx_list_push(&r->headers_out.headers); - - if (h == NULL) { - ngx_http_upstream_finalize_request(r, u, + h = u->headers_in.www_authenticate; + ph = &r->headers_out.www_authenticate; + + while (h) { + ho = ngx_list_push(&r->headers_out.headers); + + if (ho == NULL) { + ngx_http_upstream_finalize_request(r, u, NGX_HTTP_INTERNAL_SERVER_ERROR); - return NGX_OK; + return NGX_OK; + } + + *ho = *h; + ho->next = NULL; + + *ph = ho; + ph = &ho->next; + + h = h->next; } - - *h = *u->headers_in.www_authenticate; - h->next = NULL; - - r->headers_out.www_authenticate = h; } #if (NGX_HTTP_CACHE) From mdounin at mdounin.ru Wed Apr 20 22:18:56 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:56 +0300 Subject: [PATCH 16 of 20] Upstream: duplicate headers ignored or properly linked In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1650492338 -10800 # Thu Apr 21 01:05:38 2022 +0300 # Node ID f460a2f9f88d264ef6c8588eb37bcb85c48010db # Parent ab424b5e32405aeec54ccdfe38e9408209209e0a Upstream: duplicate headers ignored or properly linked. Most of the known duplicate upstream response headers are now ignored with a warning. If syntax permits multiple headers, these are now properly linked to the lists, notably Vary and WWW-Authenticate. This makes it possible to further handle such lists where it makes sense. diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -101,6 +101,9 @@ static void ngx_http_upstream_finalize_r static ngx_int_t ngx_http_upstream_process_header_line(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset); +static ngx_int_t + ngx_http_upstream_process_multi_header_lines(ngx_http_request_t *r, + ngx_table_elt_t *h, ngx_uint_t offset); static ngx_int_t ngx_http_upstream_process_content_length(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset); static ngx_int_t ngx_http_upstream_process_last_modified(ngx_http_request_t *r, @@ -226,7 +229,7 @@ static ngx_http_upstream_header_t ngx_h offsetof(ngx_http_headers_out_t, server), 0 }, { ngx_string("WWW-Authenticate"), - ngx_http_upstream_process_header_line, + ngx_http_upstream_process_multi_header_lines, offsetof(ngx_http_upstream_headers_in_t, www_authenticate), ngx_http_upstream_copy_header_line, 0, 0 }, @@ -236,7 +239,8 @@ static ngx_http_upstream_header_t ngx_h ngx_http_upstream_rewrite_location, 0, 0 }, { ngx_string("Refresh"), - ngx_http_upstream_ignore_header_line, 0, + ngx_http_upstream_process_header_line, + offsetof(ngx_http_upstream_headers_in_t, refresh), ngx_http_upstream_rewrite_refresh, 0, 0 }, { ngx_string("Set-Cookie"), @@ -2804,6 +2808,10 @@ ngx_http_upstream_process_headers(ngx_ht i = 0; } + if (h[i].hash == 0) { + continue; + } + hh = ngx_hash_find(&umcf->headers_in_hash, h[i].hash, h[i].lowcase_key, h[i].key.len); @@ -2857,6 +2865,10 @@ ngx_http_upstream_process_headers(ngx_ht i = 0; } + if (h[i].hash == 0) { + continue; + } + if (ngx_hash_find(&u->conf->hide_headers_hash, h[i].hash, h[i].lowcase_key, h[i].key.len)) { @@ -4608,10 +4620,35 @@ ngx_http_upstream_process_header_line(ng ph = (ngx_table_elt_t **) ((char *) &r->upstream->headers_in + offset); - if (*ph == NULL) { - *ph = h; - h->next = NULL; - } + if (*ph) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "upstream sent duplicate header line: \"%V: %V\", " + "previous value: \"%V: %V\", ignored", + &h->key, &h->value, + &(*ph)->key, &(*ph)->value); + h->hash = 0; + return NGX_OK; + } + + *ph = h; + h->next = NULL; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_process_multi_header_lines(ngx_http_request_t *r, + ngx_table_elt_t *h, ngx_uint_t offset) +{ + ngx_table_elt_t **ph; + + ph = (ngx_table_elt_t **) ((char *) &r->upstream->headers_in + offset); + + while (*ph) { ph = &(*ph)->next; } + + *ph = h; + h->next = NULL; return NGX_OK; } @@ -4673,6 +4710,17 @@ ngx_http_upstream_process_last_modified( u = r->upstream; + if (u->headers_in.last_modified) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "upstream sent duplicate header line: \"%V: %V\", " + "previous value: \"%V: %V\", ignored", + &h->key, &h->value, + &u->headers_in.last_modified->key, + &u->headers_in.last_modified->value); + h->hash = 0; + return NGX_OK; + } + h->next = NULL; u->headers_in.last_modified = h; u->headers_in.last_modified_time = ngx_parse_http_time(h->value.data, @@ -4842,6 +4890,18 @@ ngx_http_upstream_process_expires(ngx_ht ngx_http_upstream_t *u; u = r->upstream; + + if (u->headers_in.expires) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "upstream sent duplicate header line: \"%V: %V\", " + "previous value: \"%V: %V\", ignored", + &h->key, &h->value, + &u->headers_in.expires->key, + &u->headers_in.expires->value); + h->hash = 0; + return NGX_OK; + } + u->headers_in.expires = h; h->next = NULL; @@ -4883,6 +4943,18 @@ ngx_http_upstream_process_accel_expires( ngx_http_upstream_t *u; u = r->upstream; + + if (u->headers_in.x_accel_expires) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "upstream sent duplicate header line: \"%V: %V\", " + "previous value: \"%V: %V\", ignored", + &h->key, &h->value, + &u->headers_in.x_accel_expires->key, + &u->headers_in.x_accel_expires->value); + h->hash = 0; + return NGX_OK; + } + u->headers_in.x_accel_expires = h; h->next = NULL; @@ -4943,6 +5015,18 @@ ngx_http_upstream_process_limit_rate(ngx ngx_http_upstream_t *u; u = r->upstream; + + if (u->headers_in.x_accel_limit_rate) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "upstream sent duplicate header line: \"%V: %V\", " + "previous value: \"%V: %V\", ignored", + &h->key, &h->value, + &u->headers_in.x_accel_limit_rate->key, + &u->headers_in.x_accel_limit_rate->value); + h->hash = 0; + return NGX_OK; + } + u->headers_in.x_accel_limit_rate = h; h->next = NULL; @@ -5021,10 +5105,15 @@ static ngx_int_t ngx_http_upstream_process_connection(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { - ngx_http_upstream_t *u; + ngx_table_elt_t **ph; + ngx_http_upstream_t *u; u = r->upstream; - u->headers_in.connection = h; + ph = &u->headers_in.connection; + + while (*ph) { ph = &(*ph)->next; } + + *ph = h; h->next = NULL; if (ngx_strlcasestrn(h->value.data, h->value.data + h->value.len, @@ -5086,10 +5175,15 @@ static ngx_int_t ngx_http_upstream_process_vary(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) { - ngx_http_upstream_t *u; + ngx_table_elt_t **ph; + ngx_http_upstream_t *u; u = r->upstream; - u->headers_in.vary = h; + ph = &u->headers_in.vary; + + while (*ph) { ph = &(*ph)->next; } + + *ph = h; h->next = NULL; #if (NGX_HTTP_CACHE) diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -280,6 +280,7 @@ typedef struct { ngx_table_elt_t *last_modified; ngx_table_elt_t *location; + ngx_table_elt_t *refresh; ngx_table_elt_t *www_authenticate; ngx_table_elt_t *transfer_encoding; ngx_table_elt_t *vary; From mdounin at mdounin.ru Wed Apr 20 22:19:00 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:19:00 +0300 Subject: [PATCH 20 of 20] Headers filter: improved memory allocation error handling In-Reply-To: References: Message-ID: <6f4890227f30be59c326.1650493140@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492345 -10800 # Thu Apr 21 01:05:45 2022 +0300 # Node ID 6f4890227f30be59c3260208d0b79999b802300a # Parent 2c0188329c7d848c891feb7635fff77941449814 Headers filter: improved memory allocation error handling. diff --git a/src/http/modules/ngx_http_headers_filter_module.c b/src/http/modules/ngx_http_headers_filter_module.c --- a/src/http/modules/ngx_http_headers_filter_module.c +++ b/src/http/modules/ngx_http_headers_filter_module.c @@ -377,6 +377,7 @@ ngx_http_set_expires(ngx_http_request_t cc = ngx_list_push(&r->headers_out.headers); if (cc == NULL) { + e->hash = 0; return NGX_ERROR; } @@ -410,6 +411,8 @@ ngx_http_set_expires(ngx_http_request_t e->value.data = ngx_pnalloc(r->pool, len); if (e->value.data == NULL) { + e->hash = 0; + cc->hash = 0; return NGX_ERROR; } @@ -447,6 +450,7 @@ ngx_http_set_expires(ngx_http_request_t cc->value.data = ngx_pnalloc(r->pool, sizeof("max-age=") + NGX_TIME_T_LEN + 1); if (cc->value.data == NULL) { + cc->hash = 0; return NGX_ERROR; } From mdounin at mdounin.ru Wed Apr 20 22:18:59 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:18:59 +0300 Subject: [PATCH 19 of 20] Auth request: multiple WWW-Authenticate headers (ticket #485) In-Reply-To: References: Message-ID: <2c0188329c7d848c891f.1650493139@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492343 -10800 # Thu Apr 21 01:05:43 2022 +0300 # Node ID 2c0188329c7d848c891feb7635fff77941449814 # Parent fa9751ffe7723a11159c158078e454671e81cb87 Auth request: multiple WWW-Authenticate headers (ticket #485). When using auth_request with an upstream server which returns 401 (Unauthorized), multiple WWW-Authenticate headers from the upstream server response are now properly copied to the response. diff --git a/src/http/modules/ngx_http_auth_request_module.c b/src/http/modules/ngx_http_auth_request_module.c --- a/src/http/modules/ngx_http_auth_request_module.c +++ b/src/http/modules/ngx_http_auth_request_module.c @@ -101,7 +101,7 @@ ngx_module_t ngx_http_auth_request_modu static ngx_int_t ngx_http_auth_request_handler(ngx_http_request_t *r) { - ngx_table_elt_t *h, *ho; + ngx_table_elt_t *h, *ho, **ph; ngx_http_request_t *sr; ngx_http_post_subrequest_t *ps; ngx_http_auth_request_ctx_t *ctx; @@ -147,7 +147,9 @@ ngx_http_auth_request_handler(ngx_http_r h = sr->upstream->headers_in.www_authenticate; } - if (h) { + ph = &r->headers_out.www_authenticate; + + while (h) { ho = ngx_list_push(&r->headers_out.headers); if (ho == NULL) { return NGX_ERROR; @@ -156,7 +158,10 @@ ngx_http_auth_request_handler(ngx_http_r *ho = *h; ho->next = NULL; - r->headers_out.www_authenticate = ho; + *ph = ho; + ph = &ho->next; + + h = h->next; } return ctx->status; From mdounin at mdounin.ru Wed Apr 20 22:37:41 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:37:41 +0300 Subject: [PATCH 00 of 10] multiple headers tests In-Reply-To: References: Message-ID: Hello! Tests for the multiple headers handling patch series. -- Maxim Dounin From mdounin at mdounin.ru Wed Apr 20 22:37:42 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:37:42 +0300 Subject: [PATCH 01 of 10] Tests: tests for passing Date and Server headers In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1650492927 -10800 # Thu Apr 21 01:15:27 2022 +0300 # Node ID cc83aac5f9482dc7485d92a1e5944586774a5306 # Parent 0c50a00e67334659d58d3cf7cb81fcf5872a8285 Tests: tests for passing Date and Server headers. diff --git a/proxy_merge_headers.t b/proxy_merge_headers.t --- a/proxy_merge_headers.t +++ b/proxy_merge_headers.t @@ -10,6 +10,7 @@ use warnings; use strict; use Test::More; +use Socket qw/ CRLF /; BEGIN { use FindBin; chdir($FindBin::Bin); } @@ -21,7 +22,7 @@ use Test::Nginx; select STDERR; $| = 1; select STDOUT; $| = 1; -my $t = Test::Nginx->new()->has(qw/http proxy cache rewrite/)->plan(7) +my $t = Test::Nginx->new()->has(qw/http proxy cache rewrite/)->plan(11) ->write_file_expand('nginx.conf', <<'EOF'); %%TEST_GLOBALS%% @@ -64,6 +65,16 @@ http { proxy_pass http://127.0.0.1:8081; proxy_set_body "body"; } + + location /passdate/ { + proxy_pass http://127.0.0.1:8082; + proxy_pass_header Date; + proxy_pass_header Server; + + location /passdate/no/ { + proxy_pass http://127.0.0.1:8082; + } + } } server { @@ -80,8 +91,11 @@ http { EOF +$t->run_daemon(\&http_daemon); $t->run(); +$t->waitforsocket('127.0.0.1:' . port(8082)); + ############################################################################### like(http_get_ims('/'), qr/ims=;blah=blah;/, @@ -98,6 +112,11 @@ like(http_get('/nested/'), qr/X-Pad/, 'p unlike(http_get('/'), qr/X-Hidden/, 'proxy_hide_header inherited'); unlike(http_get('/nested/'), qr/X-Hidden/, 'proxy_hide_header nested'); +like(http_get('/passdate/'), qr/Date: passed/, 'proxy_pass_header date'); +like(http_get('/passdate/'), qr/Server: passed/, 'proxy_pass_header server'); +unlike(http_get('/passdate/no/'), qr/Date/, 'proxy_pass_header no date'); +unlike(http_get('/passdate/no/'), qr/Server/, 'proxy_pass_header no server'); + ############################################################################### sub http_get_ims { @@ -112,3 +131,45 @@ EOF } ############################################################################### + +sub http_daemon { + my $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalHost => '127.0.0.1', + LocalPort => port(8082), + Listen => 5, + Reuse => 1 + ) + or die "Can't create listening socket: $!\n"; + + local $SIG{PIPE} = 'IGNORE'; + + while (my $client = $server->accept()) { + $client->autoflush(1); + + my $headers = ''; + my $uri = ''; + + while (<$client>) { + $headers .= $_; + last if (/^\x0d?\x0a?$/); + } + + $uri = $1 if $headers =~ /^\S+\s+([^ ]+)\s+HTTP/i; + + if ($uri =~ 'no') { + print $client + 'HTTP/1.0 200 OK' . CRLF . CRLF; + + } else { + print $client + 'HTTP/1.0 200 OK' . CRLF . + 'Date: passed' . CRLF . + 'Server: passed' . CRLF . CRLF; + } + + close $client; + } +} + +############################################################################### From mdounin at mdounin.ru Wed Apr 20 22:37:43 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:37:43 +0300 Subject: [PATCH 02 of 10] Tests: fastcgi tests for combining headers In-Reply-To: References: Message-ID: <910b323534f0100d0a1c.1650494263@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492928 -10800 # Thu Apr 21 01:15:28 2022 +0300 # Node ID 910b323534f0100d0a1c8c7934a0e91c51f4e888 # Parent cc83aac5f9482dc7485d92a1e5944586774a5306 Tests: fastcgi tests for combining headers. diff --git a/fastcgi_header_params.t b/fastcgi_header_params.t --- a/fastcgi_header_params.t +++ b/fastcgi_header_params.t @@ -25,7 +25,7 @@ eval { require FCGI; }; plan(skip_all => 'FCGI not installed') if $@; plan(skip_all => 'win32') if $^O eq 'MSWin32'; -my $t = Test::Nginx->new()->has(qw/http fastcgi/)->plan(1) +my $t = Test::Nginx->new()->has(qw/http fastcgi/)->plan(4) ->write_file_expand('nginx.conf', <<'EOF'); %%TEST_GLOBALS%% @@ -59,6 +59,37 @@ EOF like(http_get_headers('/'), qr/SEE-THIS/, 'fastcgi request with many ignored headers'); +TODO: { +local $TODO = 'not yet'; + +my $r; + +$r = http(<Accept() >= 0 ) { $count++; + my $xfwd = $ENV{HTTP_X_FORWARDED_FOR} || ''; + my $cookie = $ENV{HTTP_COOKIE} || ''; + my $foo = $ENV{HTTP_FOO} || ''; + print < References: Message-ID: <0a795402009e9aaff1d2.1650494264@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492930 -10800 # Thu Apr 21 01:15:30 2022 +0300 # Node ID 0a795402009e9aaff1d2c58c7417d574e848ee43 # Parent 910b323534f0100d0a1c8c7934a0e91c51f4e888 Tests: scgi tests for combining headers. diff --git a/scgi.t b/scgi.t --- a/scgi.t +++ b/scgi.t @@ -24,7 +24,7 @@ select STDOUT; $| = 1; eval { require SCGI; }; plan(skip_all => 'SCGI not installed') if $@; -my $t = Test::Nginx->new()->has(qw/http scgi/)->plan(7) +my $t = Test::Nginx->new()->has(qw/http scgi/)->plan(10) ->write_file_expand('nginx.conf', <<'EOF'); %%TEST_GLOBALS%% @@ -81,6 +81,33 @@ like(http_get('/var?b=127.0.0.1:' . port 'scgi with variables'); like(http_get('/var?b=u'), qr/SEE-THIS/, 'scgi with variables to upstream'); +TODO: { +local $TODO = 'not yet'; + +my $r = http(<env->{HTTP_X_FORWARDED_FOR} || ''; + my $cookie = $request->env->{HTTP_COOKIE} || ''; + my $foo = $request->env->{HTTP_FOO} || ''; + $request->connection()->print(< References: Message-ID: <7769bbed69deebfb59b9.1650494265@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492931 -10800 # Thu Apr 21 01:15:31 2022 +0300 # Node ID 7769bbed69deebfb59b9f312b67788ab75d454c7 # Parent 0a795402009e9aaff1d2c58c7417d574e848ee43 Tests: uwsgi tests for combining headers. diff --git a/uwsgi.t b/uwsgi.t --- a/uwsgi.t +++ b/uwsgi.t @@ -21,7 +21,7 @@ use Test::Nginx; select STDERR; $| = 1; select STDOUT; $| = 1; -my $t = Test::Nginx->new()->has(qw/http uwsgi/)->has_daemon('uwsgi')->plan(5) +my $t = Test::Nginx->new()->has(qw/http uwsgi/)->has_daemon('uwsgi')->plan(8) ->write_file_expand('nginx.conf', <<'EOF'); %%TEST_GLOBALS%% @@ -60,7 +60,12 @@ EOF $t->write_file('uwsgi_test_app.py', <header_in() combining headers test In-Reply-To: References: Message-ID: <97a72607140290452a99.1650494268@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492936 -10800 # Thu Apr 21 01:15:36 2022 +0300 # Node ID 97a72607140290452a991ce98f3eb1d004b6eabc # Parent 01665f0b84e6cb185ec289955b0058a22459566c Tests: perl $r->header_in() combining headers test. diff --git a/perl.t b/perl.t --- a/perl.t +++ b/perl.t @@ -23,7 +23,7 @@ use Test::Nginx; select STDERR; $| = 1; select STDOUT; $| = 1; -my $t = Test::Nginx->new()->has(qw/http perl rewrite/)->plan(25) +my $t = Test::Nginx->new()->has(qw/http perl rewrite/)->plan(27) ->write_file_expand('nginx.conf', <<'EOF'); %%TEST_GLOBALS%% @@ -153,6 +153,18 @@ like(http( . 'Host: localhost' . CRLF . CRLF ), qr/xfoo: foo/, 'perl header_in unknown'); +TODO: { +local $TODO = 'not yet'; + +like(http( + 'GET / HTTP/1.0' . CRLF + . 'X-Foo: foo' . CRLF + . 'X-Foo: bar' . CRLF + . 'Host: localhost' . CRLF . CRLF +), qr/xfoo: foo, bar/, 'perl header_in unknown2'); + +} + like(http( 'GET / HTTP/1.0' . CRLF . 'Cookie: foo' . CRLF @@ -188,6 +200,13 @@ like(http( . 'Host: localhost' . CRLF . CRLF ), qr/connection: close/, 'perl header_in connection'); +like(http( + 'GET / HTTP/1.0' . CRLF + . 'Connection: close' . CRLF + . 'Connection: foo' . CRLF + . 'Host: localhost' . CRLF . CRLF +), qr/connection: close, foo/, 'perl header_in connection2'); + } # headers_out content-length tests with range filter From mdounin at mdounin.ru Wed Apr 20 22:37:50 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:37:50 +0300 Subject: [PATCH 09 of 10] Tests: tests for multiple Vary headers (ticket #1423) In-Reply-To: References: Message-ID: <1688e99291d5c4b6ed5d.1650494270@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492940 -10800 # Thu Apr 21 01:15:40 2022 +0300 # Node ID 1688e99291d5c4b6ed5dc7f86333d538acdac2f1 # Parent c5a73a2d1e3702a194ac7e7d8494acbeaf7cefe7 Tests: tests for multiple Vary headers (ticket #1423). diff --git a/proxy_cache_vary.t b/proxy_cache_vary.t --- a/proxy_cache_vary.t +++ b/proxy_cache_vary.t @@ -22,7 +22,7 @@ select STDERR; $| = 1; select STDOUT; $| = 1; my $t = Test::Nginx->new()->has(qw/http proxy cache gzip rewrite/) - ->plan(49)->write_file_expand('nginx.conf', <<'EOF'); + ->plan(52)->write_file_expand('nginx.conf', <<'EOF'); %%TEST_GLOBALS%% @@ -93,6 +93,12 @@ http { add_header Vary ",, Accept-encoding , ,"; } + location /multi { + gzip off; + add_header Vary Accept-Encoding; + add_header Vary Foo; + } + location /cold { expires max; add_header Vary $arg_vary; @@ -106,6 +112,7 @@ EOF $t->write_file('index.html', 'SEE-THIS'); $t->write_file('asterisk', 'SEE-THIS'); $t->write_file('complex', 'SEE-THIS'); +$t->write_file('multi', 'SEE-THIS'); $t->write_file('cold', 'SEE-THIS'); $t->run(); @@ -255,6 +262,18 @@ like(get('/', 'bar,foo'), qr/HIT/ms, 'no } +# Multiple Vary headers (ticket #1423). + +like(get('/multi', 'foo'), qr/MISS/ms, 'multi first'); +like(get('/multi', 'foo'), qr/HIT/ms, 'multi second'); + +TODO: { +local $TODO = 'not yet'; + +like(get('/multi', 'bar'), qr/MISS/ms, 'multi other'); + +} + # keep c->body_start when Vary changes (ticket #2029) # before 1.19.3, this prevented updating c->body_start of a main key From mdounin at mdounin.ru Wed Apr 20 22:37:47 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:37:47 +0300 Subject: [PATCH 06 of 10] Tests: perl $r->header_in("Connection") test In-Reply-To: References: Message-ID: <01665f0b84e6cb185ec2.1650494267@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492934 -10800 # Thu Apr 21 01:15:34 2022 +0300 # Node ID 01665f0b84e6cb185ec289955b0058a22459566c # Parent bde65a069a0d7ea1d7c456a9b9ed02daf9cb99fb Tests: perl $r->header_in("Connection") test. diff --git a/perl.t b/perl.t --- a/perl.t +++ b/perl.t @@ -23,7 +23,7 @@ use Test::Nginx; select STDERR; $| = 1; select STDOUT; $| = 1; -my $t = Test::Nginx->new()->has(qw/http perl rewrite/)->plan(24) +my $t = Test::Nginx->new()->has(qw/http perl rewrite/)->plan(25) ->write_file_expand('nginx.conf', <<'EOF'); %%TEST_GLOBALS%% @@ -62,6 +62,7 @@ http { $r->print("xfoo: ", $r->header_in("X-Foo"), "\n"); $r->print("cookie: ", $r->header_in("Cookie"), "\n"); $r->print("xff: ", $r->header_in("X-Forwarded-For"), "\n"); + $r->print("connection: ", $r->header_in("Connection"), "\n"); return OK; }'; @@ -178,6 +179,17 @@ like(http( . 'Host: localhost' . CRLF . CRLF ), qr/xff: foo1, foo2/, 'perl header_in xff2'); +TODO: { +local $TODO = 'not yet'; + +like(http( + 'GET / HTTP/1.0' . CRLF + . 'Connection: close' . CRLF + . 'Host: localhost' . CRLF . CRLF +), qr/connection: close/, 'perl header_in connection'); + +} + # headers_out content-length tests with range filter like(http_get('/range'), qr/Content-Length: 42.*^x{42}$/ms, From mdounin at mdounin.ru Wed Apr 20 22:37:46 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:37:46 +0300 Subject: [PATCH 05 of 10] Tests: tests for various http header variables In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1650492933 -10800 # Thu Apr 21 01:15:33 2022 +0300 # Node ID bde65a069a0d7ea1d7c456a9b9ed02daf9cb99fb # Parent 7769bbed69deebfb59b9f312b67788ab75d454c7 Tests: tests for various http header variables. diff --git a/http_headers_multi.t b/http_headers_multi.t new file mode 100644 --- /dev/null +++ b/http_headers_multi.t @@ -0,0 +1,313 @@ +#!/usr/bin/perl + +# (C) Maxim Dounin +# (C) Nginx, Inc. + +# Tests for handling of multiple http headers and access via variables. + +############################################################################### + +use warnings; +use strict; + +use Test::More; +use Socket qw/ CRLF /; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http rewrite proxy/)->plan(42); + +$t->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + server { + listen 127.0.0.1:8080; + server_name localhost; + + location / { + add_header X-Forwarded-For $http_x_forwarded_for; + add_header X-Cookie $http_cookie; + add_header X-Foo $http_foo; + + add_header X-Cookie-Foo $cookie_foo; + add_header X-Cookie-Bar $cookie_bar; + add_header X-Cookie-Bazz $cookie_bazz; + + return 204; + } + + location /s { + add_header Cache-Control foo; + add_header Cache-Control bar; + add_header Cache-Control bazz; + + add_header Link foo; + add_header Link bar; + add_header Link bazz; + + add_header Foo foo; + add_header Foo bar; + add_header Foo bazz; + + add_header X-Sent-CC $sent_http_cache_control; + add_header X-Sent-Link $sent_http_link; + add_header X-Sent-Foo $sent_http_foo; + + return 204; + } + + location /t { + add_trailer Foo foo; + add_trailer Foo bar; + add_trailer Foo bazz; + add_trailer X-Sent-Trailer-Foo $sent_trailer_foo; + + return 200 ""; + } + + location /v { + add_header X-Forwarded-For $http_x_forwarded_for; + add_header X-Cookie $http_cookie; + + add_header X-HTTP-Host $http_host; + add_header X-User-Agent $http_user_agent; + add_header X-Referer $http_referer; + add_header X-Via $http_via; + + add_header X-Content-Length $content_length; + add_header X-Content-Type $content_type; + add_header X-Host $host; + add_header X-Remote-User $remote_user; + + return 204; + } + + location /d { + return 204; + } + + location /u { + add_header X-Upstream-Set-Cookie $upstream_http_set_cookie; + add_header X-Upstream-Bar $upstream_http_bar; + + add_header X-Upstream-Cookie-Foo $upstream_cookie_foo; + add_header X-Upstream-Cookie-Bar $upstream_cookie_bar; + add_header X-Upstream-Cookie-Bazz $upstream_cookie_bazz; + + proxy_pass http://127.0.0.1:8080/backend; + } + + location /backend { + add_header Set-Cookie foo=1; + add_header Set-Cookie bar=2; + add_header Set-Cookie bazz=3; + add_header Bar foo; + add_header Bar bar; + add_header Bar bazz; + return 204; + } + } +} + +EOF + +$t->run(); + +############################################################################### + +# combining multiple headers: +# +# $http_cookie, $http_x_forwarded_for, $sent_http_cache_control, +# and $sent_http_link with special handling, other headers with +# general handling + +# request headers, $http_* + +like(get('/', map { "X-Forwarded-For: $_" } qw/ foo bar bazz /), + qr/X-Forwarded-For: foo, bar, bazz/, 'multi $http_x_forwarded_for'); +like(get('/', 'Cookie: foo=1', 'Cookie: bar=2', 'Cookie: bazz=3'), + qr/X-Cookie: foo=1; bar=2; bazz=3/, 'multi $http_cookie'); + +TODO: { +local $TODO = 'not yet'; + +like(get('/', 'Foo: foo', 'Foo: bar', 'Foo: bazz'), + qr/X-Foo: foo, bar, bazz/, 'multi $http_foo'); + +} + +# request cookies, $cookie_* + +my $r = get('/', 'Cookie: foo=1', 'Cookie: bar=2', 'Cookie: bazz=3'); + +like($r, qr/X-Cookie-Foo: 1/, '$cookie_foo'); +like($r, qr/X-Cookie-Bar: 2/, '$cookie_bar'); +like($r, qr/X-Cookie-Bazz: 3/, '$cookie_bazz'); + +# response headers, $http_* + +$r = get('/s'); + +like($r, qr/X-Sent-CC: foo, bar, bazz/, 'multi $sent_http_cache_control'); +like($r, qr/X-Sent-Link: foo, bar, bazz/, 'multi $sent_http_link'); + +TODO: { +local $TODO = 'not yet'; + +like($r, qr/X-Sent-Foo: foo, bar, bazz/, 'multi $sent_http_foo'); + +} + +# upstream response headers, $upstream_http_* + +$r = get('/u'); + +TODO: { +local $TODO = 'not yet'; + +like($r, qr/X-Upstream-Set-Cookie: foo=1, bar=2, bazz=3/, + 'multi $upstream_http_set_cookie'); +like($r, qr/X-Upstream-Bar: foo, bar, bazz/, 'multi $upstream_http_bar'); + +} + +# upstream response cookies, $upstream_cookie_* + +like($r, qr/X-Upstream-Cookie-Foo: 1/, '$upstream_cookie_foo'); +like($r, qr/X-Upstream-Cookie-Bar: 2/, '$upstream_cookie_bar'); +like($r, qr/X-Upstream-Cookie-Bazz: 3/, '$upstream_cookie_bazz'); + +# response trailers, $sent_trailer_* + +TODO: { +local $TODO = 'not yet'; + +like(get('/t'), qr/X-Sent-Trailer-Foo: foo, bar, bazz/, + 'multi $sent_trailer_foo'); + +} + +# various variables for request headers: +# +# $http_host, $http_user_agent, $http_referer +# multiple Host, User-Agent, Referer headers are invalid, but we currently +# reject only requests with multiple Host headers +# +# $http_via, $http_x_forwarded_for, $http_cookie +# multiple headers are valid + +like(get('/v'), qr/X-HTTP-Host: localhost/, '$http_host'); +like(get('/v', 'Host: foo', 'Host: bar'), + qr/400 Bad/, 'duplicate host rejected'); + +TODO: { +local $TODO = 'not yet'; + +like(get('/v', 'User-Agent: foo', 'User-Agent: bar'), + qr/X-User-Agent: foo, bar/, 'multi $http_user_agent (invalid)'); +like(get('/v', 'Referer: foo', 'Referer: bar'), + qr/X-Referer: foo, bar/, 'multi $http_referer (invalid)'); +like(get('/v', 'Via: foo', 'Via: bar', 'Via: bazz'), + qr/X-Via: foo, bar, bazz/, 'multi $http_via'); + +} + +like(get('/v', 'Cookie: foo', 'Cookie: bar', 'Cookie: bazz'), + qr/X-Cookie: foo; bar; bazz/, 'multi $http_cookie'); +like(get('/v', 'X-Forwarded-For: foo', 'X-Forwarded-For: bar', + 'X-Forwarded-For: bazz'), + qr/X-Forwarded-For: foo, bar, bazz/, 'multi $http_x_forwarded_for'); + +# other variables related to request headers: +# +# $content_length, $content_type, $host, $remote_user + +like(get('/v', 'Content-Length: 0'), + qr/X-Content-Length: 0/, '$content_length'); +like(get('/v', 'Content-Length: 0', 'Content-Length: 0'), + qr/400 Bad/, 'duplicate Content-Length rejected'); + +like(get('/v', 'Content-Type: foo'), + qr/X-Content-Type: foo/, '$content_type'); + +TODO: { +local $TODO = 'not yet'; + +like(get('/v', 'Content-Type: foo', 'Content-Type: bar'), + qr/X-Content-Type: foo, bar/, 'multi $content_type (invalid)'); + +} + +like(http("GET /v HTTP/1.0" . CRLF . CRLF), + qr/X-Host: localhost/, '$host from server_name'); +like(http("GET /v HTTP/1.0" . CRLF . "Host: foo" . CRLF . CRLF), + qr/X-Host: foo/, '$host'); +like(http("GET /v HTTP/1.0" . CRLF . "Host: foo" . CRLF . + "Host: bar" . CRLF . CRLF), + qr/400 Bad/, 'duplicate host rejected'); + +like(get('/v', 'Authorization: Basic dXNlcjpzZWNyZXQ='), + qr/X-Remote-User: user/, '$remote_user'); +like(get('/v', 'Authorization: Basic dXNlcjpzZWNyZXQ=', + 'Authorization: Basic dXNlcjpzZWNyZXQ='), + qr/400 Bad/, 'duplicate authorization rejected'); + +# request headers required to be unique: +# +# Host, If-Modified-Since, If-Unmodified-Since, If-Match, If-None-Match, +# Content-Length, Content-Range, If-Range, Transfer-Encoding, Expect, +# Authorization + +like(get('/d', 'Host: foo', 'Host: bar'), + qr/400 Bad/, 'duplicate Host rejected'); +like(get('/d', 'If-Modified-Since: foo', 'If-Modified-Since: bar'), + qr/400 Bad/, 'duplicate If-Modified-Since rejected'); +like(get('/d', 'If-Unmodified-Since: foo', 'If-Unmodified-Since: bar'), + qr/400 Bad/, 'duplicate If-Unmodified-Since rejected'); +like(get('/d', 'If-Match: foo', 'If-Match: bar'), + qr/400 Bad/, 'duplicate If-Match rejected'); +like(get('/d', 'If-None-Match: foo', 'If-None-Match: bar'), + qr/400 Bad/, 'duplicate If-None-Match rejected'); +like(get('/d', 'Content-Length: 0', 'Content-Length: 0'), + qr/400 Bad/, 'duplicate Content-Length rejected'); +like(get('/d', 'Content-Range: foo', 'Content-Range: bar'), + qr/400 Bad/, 'duplicate Content-Range rejected'); +like(get('/d', 'If-Range: foo', 'If-Range: bar'), + qr/400 Bad/, 'duplicate If-Range rejected'); +like(get('/d', 'Transfer-Encoding: foo', 'Transfer-Encoding: bar'), + qr/400 Bad/, 'duplicate Transfer-Encoding rejected'); +like(get('/d', 'Expect: foo', 'Expect: bar'), + qr/400 Bad/, 'duplicate Expect rejected'); +like(get('/d', 'Authorization: foo', 'Authorization: bar'), + qr/400 Bad/, 'duplicate Authorization rejected'); + +############################################################################### + +sub get { + my ($url, @headers) = @_; + return http( + "GET $url HTTP/1.1" . CRLF . + 'Host: localhost' . CRLF . + 'Connection: close' . CRLF . + join(CRLF, @headers) . CRLF . CRLF + ); +} + +############################################################################### From mdounin at mdounin.ru Wed Apr 20 22:37:49 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:37:49 +0300 Subject: [PATCH 08 of 10] Tests: tests for duplicate response headers In-Reply-To: References: Message-ID: # HG changeset patch # User Maxim Dounin # Date 1650492938 -10800 # Thu Apr 21 01:15:38 2022 +0300 # Node ID c5a73a2d1e3702a194ac7e7d8494acbeaf7cefe7 # Parent 97a72607140290452a991ce98f3eb1d004b6eabc Tests: tests for duplicate response headers. diff --git a/proxy_duplicate_headers.t b/proxy_duplicate_headers.t new file mode 100644 --- /dev/null +++ b/proxy_duplicate_headers.t @@ -0,0 +1,175 @@ +#!/usr/bin/perl + +# (C) Maxim Dounin + +# Test for http backend returning response with invalid and duplicate +# headers. + +############################################################################### + +use warnings; +use strict; + +use Test::More; +use Socket qw/ CRLF /; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(8); + +$t->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + server { + listen 127.0.0.1:8080; + server_name localhost; + + location / { + proxy_pass http://127.0.0.1:8081; + proxy_read_timeout 1s; + } + } +} + +EOF + +$t->run_daemon(\&http_daemon); +$t->run()->waitforsocket('127.0.0.1:' . port(8081)); + +############################################################################### + +like(http_get('/'), qr/200 OK/, 'normal'); + +TODO: { +local $TODO = 'not yet'; + +like(http_get('/invalid-length'), qr/502 Bad/, 'invalid length'); +like(http_get('/duplicate-length'), qr/502 Bad/, 'duplicate length'); +like(http_get('/unknown-transfer-encoding'), qr/502 Bad/, + 'unknown transfer encoding'); +like(http_get('/duplicate-transfer-encoding'), qr/502 Bad/, + 'duplicate transfer encoding'); +like(http_get('/length-and-transfer-encoding'), qr/502 Bad/, + 'length and transfer encoding'); +like(http_get('/transfer-encoding-and-length'), qr/502 Bad/, + 'transfer encoding and length'); + +like(http_get('/duplicate-expires'), qr/Expires: foo(?!.*bar)/s, + 'duplicate expires ignored'); + +} + +############################################################################### + +sub http_daemon { + my $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalAddr => '127.0.0.1:' . port(8081), + Listen => 5, + Reuse => 1 + ) + or die "Can't create listening socket: $!\n"; + + local $SIG{PIPE} = 'IGNORE'; + + while (my $client = $server->accept()) { + $client->autoflush(1); + + my $headers = ''; + my $uri = ''; + + while (<$client>) { + $headers .= $_; + last if (/^\x0d?\x0a?$/); + } + + $uri = $1 if $headers =~ /^\S+\s+([^ ]+)\s+HTTP/i; + + if ($uri eq '/') { + + print $client + 'HTTP/1.1 200 OK' . CRLF . + 'Connection: close' . CRLF . + 'Content-Length: 0' . CRLF . CRLF; + + } elsif ($uri eq '/invalid-length') { + + print $client + 'HTTP/1.1 200 OK' . CRLF . + 'Connection: close' . CRLF . + 'Content-Length: foo' . CRLF . CRLF; + + } elsif ($uri eq '/duplicate-length') { + + print $client + 'HTTP/1.1 200 OK' . CRLF . + 'Connection: close' . CRLF . + 'Content-Length: 0' . CRLF . + 'Content-Length: 0' . CRLF . CRLF; + + } elsif ($uri eq '/unknown-transfer-encoding') { + + print $client + 'HTTP/1.1 200 OK' . CRLF . + 'Connection: close' . CRLF . + 'Transfer-Encoding: foo' . CRLF . CRLF; + + } elsif ($uri eq '/duplicate-transfer-encoding') { + + print $client + 'HTTP/1.1 200 OK' . CRLF . + 'Connection: close' . CRLF . + 'Transfer-Encoding: chunked' . CRLF . + 'Transfer-Encoding: chunked' . CRLF . CRLF . + '0' . CRLF . CRLF; + + } elsif ($uri eq '/length-and-transfer-encoding') { + + print $client + 'HTTP/1.1 200 OK' . CRLF . + 'Connection: close' . CRLF . + 'Content-Length: 0' . CRLF . + 'Transfer-Encoding: chunked' . CRLF . CRLF . + '0' . CRLF . CRLF; + + } elsif ($uri eq '/transfer-encoding-and-length') { + + print $client + 'HTTP/1.1 200 OK' . CRLF . + 'Connection: close' . CRLF . + 'Transfer-Encoding: chunked' . CRLF . + 'Content-Length: 0' . CRLF . CRLF . + '0' . CRLF . CRLF; + + } elsif ($uri eq '/duplicate-expires') { + + print $client + 'HTTP/1.1 200 OK' . CRLF . + 'Connection: close' . CRLF . + 'Expires: foo' . CRLF . + 'Expires: bar' . CRLF . CRLF; + + } + + close $client; + } +} + +############################################################################### From mdounin at mdounin.ru Wed Apr 20 22:37:51 2022 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Thu, 21 Apr 2022 01:37:51 +0300 Subject: [PATCH 10 of 10] Tests: tests for multiple WWW-Authenticate headers (ticket #485) In-Reply-To: References: Message-ID: <11c53be196cd03ac27df.1650494271@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1650492942 -10800 # Thu Apr 21 01:15:42 2022 +0300 # Node ID 11c53be196cd03ac27df7defe5c3d297e5bbf102 # Parent 1688e99291d5c4b6ed5dc7f86333d538acdac2f1 Tests: tests for multiple WWW-Authenticate headers (ticket #485). diff --git a/auth_request.t b/auth_request.t --- a/auth_request.t +++ b/auth_request.t @@ -25,7 +25,7 @@ select STDOUT; $| = 1; my $t = Test::Nginx->new() ->has(qw/http rewrite proxy cache fastcgi auth_basic auth_request/) - ->plan(19); + ->plan(20); $t->write_file_expand('nginx.conf', <<'EOF'); @@ -129,6 +129,20 @@ http { proxy_cache_valid 1m; } + location /proxy-multi { + auth_request /auth-proxy-multi; + } + location = /auth-proxy-multi { + proxy_pass http://127.0.0.1:8080/auth-multi; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + } + location = /auth-multi { + add_header WWW-Authenticate foo always; + add_header WWW-Authenticate bar always; + return 401; + } + location /fastcgi { auth_request /auth-fastcgi; } @@ -187,6 +201,16 @@ like(http_get('/proxy-cache'), qr/ 404 / like(http_post_big('/proxy-double'), qr/ 204 /, 'proxy auth with body read'); +# Multiple WWW-Authenticate headers (ticket #485). + +TODO: { +local $TODO = 'not yet'; + +like(http_get('/proxy-multi-auth'), qr/WWW-Authenticate: foo.*bar/s, + 'multiple www-authenticate headers'); + +} + SKIP: { eval { require FCGI; }; skip 'FCGI not installed', 2 if $@; diff --git a/proxy_intercept_errors.t b/proxy_intercept_errors.t new file mode 100644 --- /dev/null +++ b/proxy_intercept_errors.t @@ -0,0 +1,105 @@ +#!/usr/bin/perl + +# (C) Maxim Dounin + +# Tests for http proxy module, proxy_intercept_errors directive. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http proxy rewrite/)->plan(4); + +$t->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + server { + listen 127.0.0.1:8080; + server_name localhost; + + location / { + proxy_pass http://127.0.0.1:8081; + proxy_intercept_errors on; + error_page 401 500 /intercepted; + } + + location = /intercepted { + return 200 "intercepted\n"; + } + } + + server { + listen 127.0.0.1:8081; + server_name localhost; + + location / { + return 404 "SEE-THIS"; + } + + location /500 { + return 500; + } + + location /auth { + add_header WWW-Authenticate foo always; + return 401; + } + + location /auth-multi { + add_header WWW-Authenticate foo always; + add_header WWW-Authenticate bar always; + return 401; + } + } +} + +EOF + +$t->run(); + +############################################################################### + +# make sure errors without error_page set are not intercepted + +like(http_get('/'), qr/SEE-THIS/, 'not intercepted'); + +# make sure errors with error_page are intercepted + +like(http_get('/500'), qr/500.*intercepted/s, 'intercepted 500'); +like(http_get('/auth'), qr/401.*WWW-Authenticate.*intercepted/s, + 'intercepted 401'); + +# make sure multiple WWW-Authenticate headers are returned +# along with intercepted response (ticket #485) + +TODO: { +local $TODO = 'not yet'; + +like(http_get('/auth-multi'), qr/401.*WWW-Authenticate: foo.*bar.*intercept/s, + 'intercepted 401 multi'); + +} + +############################################################################### From zhangfei.gao at gmail.com Thu Apr 21 03:55:00 2022 From: zhangfei.gao at gmail.com (Zhangfei Gao) Date: Thu, 21 Apr 2022 11:55:00 +0800 Subject: question about nginx start & stop In-Reply-To: References: Message-ID: On Tue, Apr 19, 2022 at 10:01 PM Maxim Dounin wrote: > > Hello! > > On Tue, Apr 19, 2022 at 12:13:45PM +0800, Zhangfei Gao wrote: > > > Hi, Maxim > > > > Thanks for the reply. > > > > On Sun, Apr 17, 2022 at 10:14 AM Maxim Dounin wrote: > > > > > > Hello! > > > > > > On Fri, Apr 15, 2022 at 03:58:52PM +0800, Zhangfei Gao wrote: > > > > > > > Hi, > > > > > > > > I have questions about nginx start and stop > > > > I am using > > > > // start > > > > sudo sbin/nginx > > > > //stop > > > > sudo sbin/nginx -s quit > > > > > > > > 1. openssl engine is init (ngx_ssl_init) twice, but openssl engine > > > > destroy function is not called. > > > > So start nginx and nginx -s quit, engine init twice but not called > > > > engine destroy. > > > > If we start and stop nginx many times, resource leakage will happen. > > > > > > OPENSSL_init_ssl manpage says: > > > > > > As of version 1.1.0 OpenSSL will automatically allocate all resources > > > that it needs so no explicit initialisation is required. Similarly it > > > will also automatically deinitialise as required. > > > > > > If there is a resource leak, this is a bug in the OpenSSL engine > > > you are testing with. It's probably up to the OpenSSL development > > > docs how to fix this properly. > > > > The openssl engine is registered with > > IMPLEMENT_DYNAMIC_BIND_FN(bind_fn) > > bind_fn() > > { > > ENGINE_set_destroy_function(e, destroy) > > ENGINE_set_finish_function(e, finish) > > } > > > > What I found is. > > /sbin/nginx > > -> bind_fn > > /sbin/nginx -s quit > > -> bind_fn > > So bind_fn is called twice, but destroy and finish are not called at all. > > > > src/core/nginx.c > > main > > ngx_ssl_init(log) -> call engine: bind_fn > > if (ngx_signal) > > return ngx_signal_process(cycle, ngx_signal); > > -> gx_os_signal_process(cycle, sig, pid) // send signal to kill > > worker process > > > > For openssl engine, what should I do to match the nginx stop. > > > > By the way, bind_fn and destroy are matched if testing with openssl > > application itself, no leakage. > > > > Any suggestions? > > The bind function is called when loading a dynamic engine and > is not expected to be matched by neither destroy nor finish. > > The finish function is called when releasing a functional > reference obtained with init (ENGINE_init() + ENGINE_finish()). > The destroy function is called when releasing a structural > reference (ENGINE_new() + ENGINE_free()). > > You shouldn't allocate resources in the bind function, but rather > only set appropriate init function to do so when needed. Thanks Maxim You are right, it is our openssl-engine issue. I found that if we registered rsa: ENGINE_set_RSA in bind_fn, destroy will not be called when nginx -s quit. Still in check. We alloc some hardware resources in bind_fn for better performance. Since the init function will be called every time. Thanks From mdounin at mdounin.ru Fri Apr 22 18:23:15 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Fri, 22 Apr 2022 21:23:15 +0300 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: <151ef936-f5d2-b681-bb54-6476c654a3ae@cdnnow.ru> References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> <151ef936-f5d2-b681-bb54-6476c654a3ae@cdnnow.ru> Message-ID: Hello! On Tue, Apr 19, 2022 at 10:59:12PM +0100, Vadim Fedorenko wrote: > On 19.04.2022 16:01, Maxim Dounin wrote: > > > > On Sun, Apr 17, 2022 at 08:50:25PM +0100, Vadim Fedorenko via nginx-devel wrote: > > > >> On 17.04.2022 02:55, Maxim Dounin wrote: > >> > >>> I'm quite sceptical about attempts to fix this by introducing > >>> various flags and reverting cacheable status back to 1. This is > >>> not how it should be fixed. > >> > >> Yeah, the solution might be a bit complicated, but I couldn't find > >> another way without breaking concept of independent header parsing. > >> Could you please suggest something if you think that current approach > >> is wrong? The ticket that this patch tries to fix is 6 years old and > >> still has discussions going on without any solution. > > > > That's basically why the ticket is still not fixed: there is no > > easy fix, and the issue is rather minor, especially compared to > > the complexity of the changes required to fix it. > > > Hmm.. I would argue on the minority of the issue, especially when we talk about > absence of compatibility with RFC and actual discussions in the ticket. Anyway. While current behaviour might be suboptimal in some use cases, it does not violate RFC, as caching is not something required to happen. So we are talking about optimizing some use cases here, and the same optimization can be easily obtained by adjusting headers in the backend response. > > IMHO, the most promising approach would be to move > > Cache-Control/Exires/X-Accel-Expires handling elsewhere, and > > handle them somewhere in ngx_http_upstream_process_headers(), > > probably along with ngx_http_file_cache_valid() as well. > > > Ok, cool, I updated my patch, now it's based on the work from Yugo that was left > without comments and is following the way you suggest. Take a look at v2, please. Apart multiple style issues, the patch fails to address the X-Accel-Expires vs. Cache-Control order issue as outlined in the ticket, that is, the response with: : X-Accel-Expires: 10 : Cache-Control: max-age=100, stale-while-revalidate=1000 results in different behaviour than the one with: : Cache-Control: max-age=100, stale-while-revalidate=1000 : X-Accel-Expires: 10 It is also seems to be completely unreadable. (Yugo's patch tried to address this, though had multiple style and readability issues as well.) -- Maxim Dounin http://mdounin.ru/ From vadim.fedorenko at cdnnow.ru Fri Apr 22 19:06:08 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Fri, 22 Apr 2022 20:06:08 +0100 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> <151ef936-f5d2-b681-bb54-6476c654a3ae@cdnnow.ru> Message-ID: <00218a02-f3e7-d1fe-04ba-1e61049f17be@cdnnow.ru> Hi Maxim! Thanks for the feedback. As you might have already seen, I sent another version of this patch which addresses some of style and readability issues as well as X-Accel-Expires issue in v3. Regarding RFC issue. RFC 7234 Section 5.1 explicitly states that Expires MUST be ignored if reponse contains Cache-Control header with max-age or s-maxage directives. That's why I said that Nginx server configured to cache responses and to rely on headers to determine the freshness violates RFC. And this is true for a subset of absolutely legitimate responses, and it was done intended to actually optimize the internal processing but not because of configuration optimizations. Hope you will have some time to review version 3 of my patch even though it might not clearly apply on top of the patchset you sent couple of days ago. Thanks, Vadim On 22.04.2022 19:23, Maxim Dounin wrote: > Hello! > > On Tue, Apr 19, 2022 at 10:59:12PM +0100, Vadim Fedorenko wrote: > >> On 19.04.2022 16:01, Maxim Dounin wrote: >>> >>> On Sun, Apr 17, 2022 at 08:50:25PM +0100, Vadim Fedorenko via nginx-devel wrote: >>> >>>> On 17.04.2022 02:55, Maxim Dounin wrote: >>>> >>>>> I'm quite sceptical about attempts to fix this by introducing >>>>> various flags and reverting cacheable status back to 1. This is >>>>> not how it should be fixed. >>>> >>>> Yeah, the solution might be a bit complicated, but I couldn't find >>>> another way without breaking concept of independent header parsing. >>>> Could you please suggest something if you think that current approach >>>> is wrong? The ticket that this patch tries to fix is 6 years old and >>>> still has discussions going on without any solution. >>> >>> That's basically why the ticket is still not fixed: there is no >>> easy fix, and the issue is rather minor, especially compared to >>> the complexity of the changes required to fix it. >>> >> Hmm.. I would argue on the minority of the issue, especially when we talk about >> absence of compatibility with RFC and actual discussions in the ticket. Anyway. > > While current behaviour might be suboptimal in some use cases, 34it > does not violate RFC, as caching is not something required to > happen. So we are talking about optimizing some use cases here, > and the same optimization can be easily obtained by adjusting > headers in the backend response. > >>> IMHO, the most promising approach would be to move >>> Cache-Control/Exires/X-Accel-Expires handling elsewhere, and >>> handle them somewhere in ngx_http_upstream_process_headers(), >>> probably along with ngx_http_file_cache_valid() as well. >>> >> Ok, cool, I updated my patch, now it's based on the work from Yugo that was left >> without comments and is following the way you suggest. Take a look at v2, please. > > Apart multiple style issues, the patch fails to address the > X-Accel-Expires vs. Cache-Control order issue as outlined in the > ticket, that is, the response with: > > : X-Accel-Expires: 10 > : Cache-Control: max-age=100, stale-while-revalidate=1000 > > results in different behaviour than the one with: > > : Cache-Control: max-age=100, stale-while-revalidate=1000 > : X-Accel-Expires: 10 > > It is also seems to be completely unreadable. (Yugo's patch tried > to address this, though had multiple style and readability issues > as well.) > From xeioex at nginx.com Sat Apr 23 00:04:42 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Sat, 23 Apr 2022 00:04:42 +0000 Subject: [njs] Fixed Array.prototype.sort() when arr size is changed in a comparator. Message-ID: details: https://hg.nginx.org/njs/rev/9424f42b7266 branches: changeset: 1839:9424f42b7266 user: Dmitry Volyntsev date: Fri Apr 22 17:02:28 2022 -0700 description: Fixed Array.prototype.sort() when arr size is changed in a comparator. This fixed #468 issue on Github. diffstat: src/njs_array.c | 2 +- src/test/njs_unit_test.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletions(-) diffs (25 lines): diff -r 6b226ed1b25d -r 9424f42b7266 src/njs_array.c --- a/src/njs_array.c Thu Apr 14 16:07:34 2022 -0700 +++ b/src/njs_array.c Fri Apr 22 17:02:28 2022 -0700 @@ -2696,7 +2696,7 @@ slow_path: goto exception; } - if (njs_fast_path(fast_path)) { + if (njs_fast_path(fast_path && njs_is_fast_array(this))) { array = njs_array(this); start = array->start; diff -r 6b226ed1b25d -r 9424f42b7266 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Thu Apr 14 16:07:34 2022 -0700 +++ b/src/test/njs_unit_test.c Fri Apr 22 17:02:28 2022 -0700 @@ -6989,6 +6989,9 @@ static njs_unit_test_t njs_test[] = { njs_str("[1,2].sort(1)"), njs_str("TypeError: comparefn must be callable or undefined") }, + { njs_str("var a = [1,2]; a.sort(() => {a.length = 65535}); a.length"), + njs_str("65535") }, + /* Array.prototype.keys() Array.prototype.values() From xeioex at nginx.com Sat Apr 23 00:04:44 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Sat, 23 Apr 2022 00:04:44 +0000 Subject: [njs] Fixed Array.prototype.slice() with slow "this" argument. Message-ID: details: https://hg.nginx.org/njs/rev/3fec53d722ef branches: changeset: 1840:3fec53d722ef user: Dmitry Volyntsev date: Fri Apr 22 17:02:36 2022 -0700 description: Fixed Array.prototype.slice() with slow "this" argument. Previously, when "this" argument was not a fast array, but the "deleted" array was a fast array, the "deleted" array may be left in uninitialized state if "this" argument had gaps. This fix is to ensure that "deleted" is properly initialized. This fixes #485 issue on Github. diffstat: src/njs_array.c | 5 +++++ src/test/njs_unit_test.c | 9 +++++++++ 2 files changed, 14 insertions(+), 0 deletions(-) diffs (34 lines): diff -r 9424f42b7266 -r 3fec53d722ef src/njs_array.c --- a/src/njs_array.c Fri Apr 22 17:02:28 2022 -0700 +++ b/src/njs_array.c Fri Apr 22 17:02:36 2022 -0700 @@ -1284,6 +1284,11 @@ njs_array_prototype_splice(njs_vm_t *vm, if (njs_slow_path(ret == NJS_ERROR)) { return ret; } + + } else { + if (deleted->object.fast_array) { + njs_set_invalid(&deleted->start[i]); + } } } diff -r 9424f42b7266 -r 3fec53d722ef src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Fri Apr 22 17:02:28 2022 -0700 +++ b/src/test/njs_unit_test.c Fri Apr 22 17:02:36 2022 -0700 @@ -4869,6 +4869,15 @@ static njs_unit_test_t njs_test[] = "Array.prototype.splice.call(obj, 2**53-2, 0, 'C');"), njs_str("TypeError: Invalid length") }, + { njs_str("var a = {1: 'B', length: 2};" + "Array.prototype.splice.call(a, 0)"), + njs_str(",B") }, + + { njs_str("var a = new Uint8Array();" + "a.__proto__ = [1,2,3];" + "a.splice(0)"), + njs_str(",,") }, + { njs_str("var a = []; a.reverse()"), njs_str("") }, From mdounin at mdounin.ru Sun Apr 24 04:55:17 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sun, 24 Apr 2022 07:55:17 +0300 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: <00218a02-f3e7-d1fe-04ba-1e61049f17be@cdnnow.ru> References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> <151ef936-f5d2-b681-bb54-6476c654a3ae@cdnnow.ru> <00218a02-f3e7-d1fe-04ba-1e61049f17be@cdnnow.ru> Message-ID: Hello! On Fri, Apr 22, 2022 at 08:06:08PM +0100, Vadim Fedorenko via nginx-devel wrote: > Hi Maxim! > > Thanks for the feedback. As you might have already seen, I sent another version > of this patch which addresses some of style and readability issues as well as > X-Accel-Expires issue in v3. Ah, sorry, I've missed that you've removed the "u->headers_in.x_accel_expires != NULL" check in the ngx_http_upstream_process_cache_control(). So the stale-while-revalidate and stale-if-error Cache-Control directive are always applied now, whenever X-Accel-Expires is present or not. Not sure this is a good approach: simply ignoring Cache-Control if X-Accel-Expires is present might be a better and easier to understand solution. The basic idea behind X-Accel-Expires is that site administrators might want to provide distinct caching parameters to caches they control (and can purge or force-update when needed), and these caching parameters are different from caching parameters sent to external caches. Still using some caching parameters from Cache-Control does not seem to align well with this usage. On the other hand, as per RFC 2616/7234, the Expires header is only expected to be ignored when "Cache-Control: max-age" is present, but not other Cache-Control directives. So probably you are right and this is correct approach to use. > Regarding RFC issue. RFC 7234 Section 5.1 explicitly states that Expires MUST be > ignored if reponse contains Cache-Control header with max-age or s-maxage > directives. That's why I said that Nginx server configured to cache responses > and to rely on headers to determine the freshness violates RFC. And this is true > for a subset of absolutely legitimate responses, and it was done intended to > actually optimize the internal processing but not because of configuration > optimizations. Sure. Though it is also completely legitimate to don't cache a response, regardless of the particular headers. Switching of caching for any reason is generally safe, and makes further RFC 7234 requirements void. Rather, I would say that nginx violates RFC by caching a response based on "Cache-Control: max-age" when there is an Expires header which prevents caching, as nginx does not support the Age header and therefore can incorrectly cache an expired response. > Hope you will have some time to review version 3 of my patch even though it > might not clearly apply on top of the patchset you sent couple of days ago. Unfortunately, style and readability issues are still there. Review below. (Note that it might be a good idea to keep patches in a single thread. When sending with "hg email", the "--in-reply-to" option ensures proper threading.) : # HG changeset patch : # User Vadim Fedorenko : # Date 1650406016 -10800 : # Wed Apr 20 01:06:56 2022 +0300 : # Node ID e04dac22328020cf8d8abcc4863b982b513b0c80 : # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b : Upstream: prioritise X-Accel-Expire over Cache-Control and Expire Note missing "s" in "Expires". Style: should be a trailing dot. : RFC7234 explicitly says that cache recipient MUST ignore Expires : header if response includes Cache-Control header with max-age or : s-maxage directives. Previously Cache-Control was ignored if it : was after Expires in reply headers. Note that this is an inaccurate and misleading statement: Cache-Control wasn't ignored. The only specific case to address is when the Expires header comes first and disables caching. : At the same time documentation states that there is special header : X-Accel-Expires that controls the behavior of nginx and should be : prioritized over other cache-specific headers and this patch : implements this priority. Again, the X-Accel-Expires header is already prioritized over any other cache control headers. : More informantion could be found in ticket #964. Typo: "information". References in the "(ticket #964)" form are automatically linked in Trac and generally recommended unless there are specific reasons to avoid linking. : --- : src/http/ngx_http_upstream.c | 62 +++++++++++++++++++++++++----------- : src/http/ngx_http_upstream.h | 6 ++++ : 2 files changed, 50 insertions(+), 18 deletions(-) : : diff -r a736a7a613ea -r e04dac223280 src/http/ngx_http_upstream.c : --- a/src/http/ngx_http_upstream.c Tue Feb 08 17:35:27 2022 +0300 : +++ b/src/http/ngx_http_upstream.c Wed Apr 20 01:06:56 2022 +0300 : @@ -2350,6 +2350,32 @@ : : : static void : +ngx_http_upstream_validate_cache_headers(ngx_http_request_t *r, ngx_http_upstream_t *u) : +{ : + ngx_http_upstream_headers_in_t *uh = &u->headers_in; : + if (uh->x_accel_expires != NULL && : + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_EXPIRES)) { Style issues: more than 80 characters in a line, no empty line between variable definition and code, less than two spaces between type and variable name, variable initialization should be on a separate line, two spaces before "&&", operator should be on the continuation line, "{" should be on its own line, no function declaration, function definition should follow first usage unless there are specific reasons to place it elsewhere. Also, we don't generally use ngx_http_upstream_headers_in_t local variables, but rather use direct references via u->headers_in. : + u->cacheable = uh->x_accel_expires_c; : + r->cache->valid_sec = uh->x_accel_expires_n; This uses r->cache without appropriate conditional compilation directives, so will result in compilation errors without HTTP cache compiled in ("./configure --without-http-cache"). : + return; : + } : + : + if (uh->cache_control.elts != NULL && : + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_CACHE_CONTROL)) { : + u->cacheable = uh->cache_control_c; : + r->cache->valid_sec = uh->cache_control_n; This seems to be incorrect, given that something like Cache-Control: public Expires: is exactly equivalent to just "Expires: ", without Cache-Control at all. : + return; : + } : + : + if (uh->expires != NULL && : + !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_EXPIRES)) { : + u->cacheable = uh->expires_c; : + r->cache->valid_sec = uh->expires_n; : + } : +} : + : + : +static void : ngx_http_upstream_process_header(ngx_http_request_t *r, ngx_http_upstream_t *u) : { : ssize_t n; : @@ -2468,6 +2494,11 @@ : : continue; : } : +#if (NGX_HTTP_CACHE) Conditional compilation here would result in a function which is never used without HTTP cache compiled in, and hence "unused function" warnings. : + if (u->cacheable) { : + ngx_http_upstream_validate_cache_headers(r, u); : + } : +#endif Caching also needs to be properly adjusted somewhere in ngx_http_upstream_intercept_errors(), as it does not call ngx_http_upstream_process_headers(). : : break; : } : @@ -4735,10 +4766,6 @@ : return NGX_OK; : } : : - if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { : - return NGX_OK; : - } : - : start = h->value.data; : last = start + h->value.len; : : @@ -4746,7 +4773,7 @@ : || ngx_strlcasestrn(start, last, (u_char *) "no-store", 8 - 1) != NULL : || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) : { : - u->cacheable = 0; : + u->headers_in.cache_control_c = 0; : return NGX_OK; : } : : @@ -4771,16 +4798,15 @@ : continue; : } : : - u->cacheable = 0; : return NGX_OK; : } : : if (n == 0) { : - u->cacheable = 0; : return NGX_OK; : } : : - r->cache->valid_sec = ngx_time() + n; : + u->headers_in.cache_control_c = 1; : + u->headers_in.cache_control_n = ngx_time() + n; : } : : p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", : @@ -4856,18 +4882,18 @@ : return NGX_OK; : } : : - if (r->cache->valid_sec != 0) { : + expires = ngx_parse_http_time(h->value.data, h->value.len); : + : + if (expires == NGX_ERROR || expires < ngx_time()) { : return NGX_OK; : } : : - expires = ngx_parse_http_time(h->value.data, h->value.len); : - : - if (expires == NGX_ERROR || expires < ngx_time()) { : - u->cacheable = 0; : - return NGX_OK; : - } : - : - r->cache->valid_sec = expires; : + if (u->headers_in.expires_c) { : + expires = ngx_min(expires, u->headers_in.expires_n); : + } The patch series recently posted will take care of duplicate headers, there is no need to introduce additional ad-hoc solutions. : + u->headers_in.expires_n = expires; : + u->headers_in.expires_c = 1; : + : } : #endif : : @@ -4906,14 +4932,12 @@ : : switch (n) { : case 0: : - u->cacheable = 0; : - /* fall through */ : - : case NGX_ERROR: : return NGX_OK; : : default: : - r->cache->valid_sec = ngx_time() + n; : + u->headers_in.x_accel_expires_c = 1; : + u->headers_in.x_accel_expires_n = ngx_time() + n; : return NGX_OK; : } : } : @@ -4924,7 +4948,8 @@ : n = ngx_atoi(p, len); : : if (n != NGX_ERROR) { : - r->cache->valid_sec = n; : + u->headers_in.x_accel_expires_c = 1; : + u->headers_in.x_accel_expires_n = ngx_time() + n; : } : } : #endif : diff -r a736a7a613ea -r e04dac223280 src/http/ngx_http_upstream.h : --- a/src/http/ngx_http_upstream.h Tue Feb 08 17:35:27 2022 +0300 : +++ b/src/http/ngx_http_upstream.h Wed Apr 20 01:06:56 2022 +0300 : @@ -294,9 +294,15 @@ : : off_t content_length_n; : time_t last_modified_time; : + ngx_int_t cache_control_n; : + ngx_int_t expires_n; : + ngx_int_t x_accel_expires_n; : : unsigned connection_close:1; : unsigned chunked:1; : + unsigned cache_control_c; : + unsigned expires_c; : + unsigned x_accel_expires_c; : } ngx_http_upstream_headers_in_t; : : As far as I can tell, proper behaviour, assuming we parse cache control extensions independently of X-Accel-Expires, can be implemented by using just one flag. Patch below, review and testing appreciated: # HG changeset patch # User Maxim Dounin # Date 1650775261 -10800 # Sun Apr 24 07:41:01 2022 +0300 # Node ID f9c6d561b510e6008bfb4d7989f358dba33b38cd # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b Upstream: fixed X-Accel-Expires/Cache-Control/Expires handling. Previously, if caching was disabled due to Expires in the past, nginx failed to cache the response even if it was cacheable as per subsequently parsed Cache-Control header (ticket #964). Similarly, if caching was disabled due to Expires in the past, "Cache-Control: no-cache" or "Cache-Control: max-age=0", caching was not used if it was cacheable as per subsequently parsed X-Accel-Expires header. Fix is to avoid disabling caching immediately after parsing Expires in the past, but rather set a flag which is later checked by ngx_http_upstream_process_headers() (and cleared by "Cache-Control: max-age" and X-Accel-Expires). Additionally, now X-Accel-Expires does not prevent parsing of cache control extensions, notably stale-while-revalidate and stale-if-error. This ensures that order of the X-Accel-Expires and Cache-Control headers is not important. Prodded by Vadim Fedorenko and Yugo Horie. diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -2697,6 +2697,10 @@ ngx_http_upstream_intercept_errors(ngx_h if (r->cache) { + if (u->headers_in.no_cache) { + u->cacheable = 0; + } + if (u->cacheable) { time_t valid; @@ -2791,6 +2795,10 @@ ngx_http_upstream_process_headers(ngx_ht umcf = ngx_http_get_module_main_conf(r, ngx_http_upstream_module); + if (u->headers_in.no_cache) { + u->cacheable = 0; + } + if (u->headers_in.x_accel_redirect && !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_REDIRECT)) { @@ -4735,18 +4743,18 @@ ngx_http_upstream_process_cache_control( return NGX_OK; } - if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { - return NGX_OK; - } - start = h->value.data; last = start + h->value.len; + if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { + goto extensions; + } + if (ngx_strlcasestrn(start, last, (u_char *) "no-cache", 8 - 1) != NULL || ngx_strlcasestrn(start, last, (u_char *) "no-store", 8 - 1) != NULL || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) { - u->cacheable = 0; + u->headers_in.no_cache = 1; return NGX_OK; } @@ -4776,12 +4784,15 @@ ngx_http_upstream_process_cache_control( } if (n == 0) { - u->cacheable = 0; + u->headers_in.no_cache = 1; return NGX_OK; } r->cache->valid_sec = ngx_time() + n; - } + u->headers_in.no_cache = 0; + } + +extensions: p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", 23 - 1); @@ -4863,7 +4874,7 @@ ngx_http_upstream_process_expires(ngx_ht expires = ngx_parse_http_time(h->value.data, h->value.len); if (expires == NGX_ERROR || expires < ngx_time()) { - u->cacheable = 0; + u->headers_in.no_cache = 1; return NGX_OK; } @@ -4914,6 +4925,7 @@ ngx_http_upstream_process_accel_expires( default: r->cache->valid_sec = ngx_time() + n; + u->headers_in.no_cache = 0; return NGX_OK; } } @@ -4925,6 +4937,7 @@ ngx_http_upstream_process_accel_expires( if (n != NGX_ERROR) { r->cache->valid_sec = n; + u->headers_in.no_cache = 0; } } #endif diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -297,6 +297,7 @@ typedef struct { unsigned connection_close:1; unsigned chunked:1; + unsigned no_cache:1; } ngx_http_upstream_headers_in_t; -- Maxim Dounin http://mdounin.ru/ From shanlei at asiainfo.com Sun Apr 24 08:33:04 2022 From: shanlei at asiainfo.com (shanlei at asiainfo.com) Date: Sun, 24 Apr 2022 16:33:04 +0800 Subject: patch: call modules' exit_process in reverse order Message-ID: <564de12e05a6d3b229e395855712255f@asiainfo.com> # HG changeset patch # User stdanley # Date 1650788278 -28800 # Sun Apr 24 16:17:58 2022 +0800 # Node ID 522acbe88486d027383075c8208edd6fcc0a3aa6 # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b patch: call modules' exit_process in reverse order to solve module dependency. We once have developed a module which depends on ngx_thread_pool module. but nginx hungs when began to shutdown. trace shows that ngx_thread_pool cant finish "exit_process" because our module is using a thread, our module must call "exit_process" before ngx_thread_pool. so we present this patch to solve modules' dependency issue. diff -r a736a7a613ea -r 522acbe88486 src/os/unix/ngx_process_cycle.c --- a/src/os/unix/ngx_process_cycle.c Tue Feb 08 17:35:27 2022 +0300 +++ b/src/os/unix/ngx_process_cycle.c Sun Apr 24 16:17:58 2022 +0800 @@ -300,10 +300,13 @@ ngx_process_events_and_timers(cycle); if (ngx_terminate || ngx_quit) { - - for (i = 0; cycle->modules[i]; i++) { - if (cycle->modules[i]->exit_process) { - cycle->modules[i]->exit_process(cycle); + ngx_int_t j, k = 0; + //for (i = 0; cycle->modules[i]; i++) { + for (j = 0; cycle->modules[j]; j++, k++) + ; + for (j = k - 1; j >= 0; j--) { + if (cycle->modules[j]->exit_process) { + cycle->modules[j]->exit_process(cycle); } } @@ -950,10 +953,14 @@ { ngx_uint_t i; ngx_connection_t *c; + ngx_int_t j, k=0; - for (i = 0; cycle->modules[i]; i++) { - if (cycle->modules[i]->exit_process) { - cycle->modules[i]->exit_process(cycle); + //for (i = 0; cycle->modules[i]; i++) { + for (j = 0; cycle->modules[j]; j++, k++) + ; + for (j = k - 1; j >= 0; j--) { + if (cycle->modules[j]->exit_process) { + cycle->modules[j]->exit_process(cycle); } } From mdounin at mdounin.ru Sun Apr 24 15:42:47 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sun, 24 Apr 2022 18:42:47 +0300 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> <151ef936-f5d2-b681-bb54-6476c654a3ae@cdnnow.ru> <00218a02-f3e7-d1fe-04ba-1e61049f17be@cdnnow.ru> Message-ID: Hello! On Sun, Apr 24, 2022 at 07:55:17AM +0300, Maxim Dounin wrote: [...] > As far as I can tell, proper behaviour, assuming we parse cache > control extensions independently of X-Accel-Expires, can be > implemented by using just one flag. No, that's wrong, as with just one flag it wouldn't be possible to correctly disable caching of responses with: Cache-Control: private Cache-Control: max-age=10 So it needs at least two flags. Updated patch below, review and testing appreciated. # HG changeset patch # User Maxim Dounin # Date 1650814681 -10800 # Sun Apr 24 18:38:01 2022 +0300 # Node ID 940ba4317a97c72d1ee6700cbf58a543fee04c7a # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b Upstream: fixed X-Accel-Expires/Cache-Control/Expires handling. Previously, if caching was disabled due to Expires in the past, nginx failed to cache the response even if it was cacheable as per subsequently parsed Cache-Control header (ticket #964). Similarly, if caching was disabled due to Expires in the past, "Cache-Control: no-cache" or "Cache-Control: max-age=0", caching was not used if it was cacheable as per subsequently parsed X-Accel-Expires header. Fix is to avoid disabling caching immediately after parsing Expires in the past or Cache-Control, but rather set flags which are later checked by ngx_http_upstream_process_headers() (and cleared by "Cache-Control: max-age" and X-Accel-Expires). Additionally, now X-Accel-Expires does not prevent parsing of cache control extensions, notably stale-while-revalidate and stale-if-error. This ensures that order of the X-Accel-Expires and Cache-Control headers is not important. Prodded by Vadim Fedorenko and Yugo Horie. diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -2697,6 +2697,10 @@ ngx_http_upstream_intercept_errors(ngx_h if (r->cache) { + if (u->headers_in.no_cache || u->headers_in.expired) { + u->cacheable = 0; + } + if (u->cacheable) { time_t valid; @@ -2791,6 +2795,10 @@ ngx_http_upstream_process_headers(ngx_ht umcf = ngx_http_get_module_main_conf(r, ngx_http_upstream_module); + if (u->headers_in.no_cache || u->headers_in.expired) { + u->cacheable = 0; + } + if (u->headers_in.x_accel_redirect && !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_REDIRECT)) { @@ -4735,18 +4743,18 @@ ngx_http_upstream_process_cache_control( return NGX_OK; } - if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { - return NGX_OK; - } - start = h->value.data; last = start + h->value.len; + if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { + goto extensions; + } + if (ngx_strlcasestrn(start, last, (u_char *) "no-cache", 8 - 1) != NULL || ngx_strlcasestrn(start, last, (u_char *) "no-store", 8 - 1) != NULL || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) { - u->cacheable = 0; + u->headers_in.no_cache = 1; return NGX_OK; } @@ -4776,12 +4784,15 @@ ngx_http_upstream_process_cache_control( } if (n == 0) { - u->cacheable = 0; + u->headers_in.no_cache = 1; return NGX_OK; } r->cache->valid_sec = ngx_time() + n; - } + u->headers_in.expired = 0; + } + +extensions: p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", 23 - 1); @@ -4863,7 +4874,7 @@ ngx_http_upstream_process_expires(ngx_ht expires = ngx_parse_http_time(h->value.data, h->value.len); if (expires == NGX_ERROR || expires < ngx_time()) { - u->cacheable = 0; + u->headers_in.expired = 1; return NGX_OK; } @@ -4914,6 +4925,8 @@ ngx_http_upstream_process_accel_expires( default: r->cache->valid_sec = ngx_time() + n; + u->headers_in.no_cache = 0; + u->headers_in.expired = 0; return NGX_OK; } } @@ -4925,6 +4938,8 @@ ngx_http_upstream_process_accel_expires( if (n != NGX_ERROR) { r->cache->valid_sec = n; + u->headers_in.no_cache = 0; + u->headers_in.expired = 0; } } #endif diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -297,6 +297,8 @@ typedef struct { unsigned connection_close:1; unsigned chunked:1; + unsigned no_cache:1; + unsigned expired:1; } ngx_http_upstream_headers_in_t; -- Maxim Dounin http://mdounin.ru/ From yugo-horie at jocdn.co.jp Mon Apr 25 12:27:05 2022 From: yugo-horie at jocdn.co.jp (Yugo Horie) Date: Mon, 25 Apr 2022 21:27:05 +0900 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> <151ef936-f5d2-b681-bb54-6476c654a3ae@cdnnow.ru> <00218a02-f3e7-d1fe-04ba-1e61049f17be@cdnnow.ru> Message-ID: Hello, I appriciate deeply with two of your cooperation. Maxim's patch looks good to see but it seems to be a bit weird about the behavior of the r->cache->valid_sec. In case of the `X-Accel-Expires: 0` header comes first than the `Cache-Control: max-age=10` header, the r->cache->valid_sec could not be overwritten by the process_accel_expires which has a weird switch (if it is case 0, it would be fall through and return NGX_OK) statements, and then it would be overwritten by the process_cache_control(thus Cache-Control:max-age). In other word, it cannot kick out the max-age parsing with your brand new extension flag because it cannot clear if statements of r->cache-valid_sec !=0. In contrasts, when the Cache-Control is first, it would be overwritten by X-Accel-Expires. I consider this is the right behavior. Thanks, Yugo Horie On Mon, Apr 25, 2022 at 0:43 Maxim Dounin wrote: > Hello! > > On Sun, Apr 24, 2022 at 07:55:17AM +0300, Maxim Dounin wrote: > > [...] > > > As far as I can tell, proper behaviour, assuming we parse cache > > control extensions independently of X-Accel-Expires, can be > > implemented by using just one flag. > > No, that's wrong, as with just one flag it wouldn't be possible to > correctly disable caching of responses with: > > Cache-Control: private > Cache-Control: max-age=10 > > So it needs at least two flags. Updated patch below, review and > testing appreciated. > > # HG changeset patch > # User Maxim Dounin > # Date 1650814681 -10800 > # Sun Apr 24 18:38:01 2022 +0300 > # Node ID 940ba4317a97c72d1ee6700cbf58a543fee04c7a > # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b > Upstream: fixed X-Accel-Expires/Cache-Control/Expires handling. > > Previously, if caching was disabled due to Expires in the past, nginx > failed to cache the response even if it was cacheable as per subsequently > parsed Cache-Control header (ticket #964). > > Similarly, if caching was disabled due to Expires in the past, > "Cache-Control: no-cache" or "Cache-Control: max-age=0", caching was not > used if it was cacheable as per subsequently parsed X-Accel-Expires header. > > Fix is to avoid disabling caching immediately after parsing Expires in > the past or Cache-Control, but rather set flags which are later checked by > ngx_http_upstream_process_headers() (and cleared by "Cache-Control: > max-age" > and X-Accel-Expires). > > Additionally, now X-Accel-Expires does not prevent parsing of cache control > extensions, notably stale-while-revalidate and stale-if-error. This > ensures that order of the X-Accel-Expires and Cache-Control headers is not > important. > > Prodded by Vadim Fedorenko and Yugo Horie. > > diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c > --- a/src/http/ngx_http_upstream.c > +++ b/src/http/ngx_http_upstream.c > @@ -2697,6 +2697,10 @@ ngx_http_upstream_intercept_errors(ngx_h > > if (r->cache) { > > + if (u->headers_in.no_cache || u->headers_in.expired) { > + u->cacheable = 0; > + } > + > if (u->cacheable) { > time_t valid; > > @@ -2791,6 +2795,10 @@ ngx_http_upstream_process_headers(ngx_ht > > umcf = ngx_http_get_module_main_conf(r, ngx_http_upstream_module); > > + if (u->headers_in.no_cache || u->headers_in.expired) { > + u->cacheable = 0; > + } > + > if (u->headers_in.x_accel_redirect > && !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_REDIRECT)) > { > @@ -4735,18 +4743,18 @@ ngx_http_upstream_process_cache_control( > return NGX_OK; > } > > - if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != > NULL) { > - return NGX_OK; > - } > - > start = h->value.data; > last = start + h->value.len; > > + if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != > NULL) { > + goto extensions; > + } > + > if (ngx_strlcasestrn(start, last, (u_char *) "no-cache", 8 - 1) != > NULL > || ngx_strlcasestrn(start, last, (u_char *) "no-store", 8 - 1) != > NULL > || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != > NULL) > { > - u->cacheable = 0; > + u->headers_in.no_cache = 1; > return NGX_OK; > } > > @@ -4776,12 +4784,15 @@ ngx_http_upstream_process_cache_control( > } > > if (n == 0) { > - u->cacheable = 0; > + u->headers_in.no_cache = 1; > return NGX_OK; > } > > r->cache->valid_sec = ngx_time() + n; > - } > + u->headers_in.expired = 0; > + } > + > +extensions: > > p = ngx_strlcasestrn(start, last, (u_char *) > "stale-while-revalidate=", > 23 - 1); > @@ -4863,7 +4874,7 @@ ngx_http_upstream_process_expires(ngx_ht > expires = ngx_parse_http_time(h->value.data, h->value.len); > > if (expires == NGX_ERROR || expires < ngx_time()) { > - u->cacheable = 0; > + u->headers_in.expired = 1; > return NGX_OK; > } > > @@ -4914,6 +4925,8 @@ ngx_http_upstream_process_accel_expires( > > default: > r->cache->valid_sec = ngx_time() + n; > + u->headers_in.no_cache = 0; > + u->headers_in.expired = 0; > return NGX_OK; > } > } > @@ -4925,6 +4938,8 @@ ngx_http_upstream_process_accel_expires( > > if (n != NGX_ERROR) { > r->cache->valid_sec = n; > + u->headers_in.no_cache = 0; > + u->headers_in.expired = 0; > } > } > #endif > diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h > --- a/src/http/ngx_http_upstream.h > +++ b/src/http/ngx_http_upstream.h > @@ -297,6 +297,8 @@ typedef struct { > > unsigned connection_close:1; > unsigned chunked:1; > + unsigned no_cache:1; > + unsigned expired:1; > } ngx_http_upstream_headers_in_t; > > > > -- > Maxim Dounin > http://mdounin.ru/ > _______________________________________________ > nginx-devel mailing list -- nginx-devel at nginx.org > To unsubscribe send an email to nginx-devel-leave at nginx.org > -------------- next part -------------- An HTML attachment was scrubbed... URL: From vadim.fedorenko at cdnnow.ru Mon Apr 25 21:03:20 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Mon, 25 Apr 2022 22:03:20 +0100 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> <151ef936-f5d2-b681-bb54-6476c654a3ae@cdnnow.ru> <00218a02-f3e7-d1fe-04ba-1e61049f17be@cdnnow.ru> Message-ID: <99ff7b52-608e-21bb-7a01-11e117fd5979@cdnnow.ru> Hi Yugo! I will disagree with your comment. If X-Accel-Expires has value 0 then r->cacheable will be zeroed and no caching will occur anyway and the valid_sec will never be checked. Overall the patch looks like a simplified version of what we already had, which is great! Best wishes, Vadim On 25.04.2022 13:27, Yugo Horie wrote: > Hello, > I appriciate deeply with two of your cooperation. > > Maxim's patch looks good to see but it seems to be a bit weird about the > behavior of the r->cache->valid_sec. > In case of the `X-Accel-Expires: 0` header comes first than the `Cache-Control: > max-age=10` header, the r->cache->valid_sec could not be overwritten by the > process_accel_expires which has a weird switch (if it is case 0, it would be > fall through and return NGX_OK) statements, and then it would be overwritten by > the process_cache_control(thus Cache-Control:max-age). In other word, it cannot > kick out the max-age parsing with your brand new extension flag because it > cannot clear if statements of r->cache-valid_sec !=0. > > In contrasts, when the Cache-Control is first, it would be overwritten by > X-Accel-Expires. I consider this is the right behavior. > > Thanks, > Yugo Horie > > On Mon, Apr 25, 2022 at 0:43 Maxim Dounin > wrote: > > Hello! > > On Sun, Apr 24, 2022 at 07:55:17AM +0300, Maxim Dounin wrote: > > [...] > > > As far as I can tell, proper behaviour, assuming we parse cache > > control extensions independently of X-Accel-Expires, can be > > implemented by using just one flag. > > No, that's wrong, as with just one flag it wouldn't be possible to > correctly disable caching of responses with: > > Cache-Control: private > Cache-Control: max-age=10 > > So it needs at least two flags.  Updated patch below, review and > testing appreciated. > > # HG changeset patch > # User Maxim Dounin > > # Date 1650814681 -10800 > #      Sun Apr 24 18:38:01 2022 +0300 > # Node ID 940ba4317a97c72d1ee6700cbf58a543fee04c7a > # Parent  a736a7a613ea6e182ff86fbadcb98bb0f8891c0b > Upstream: fixed X-Accel-Expires/Cache-Control/Expires handling. > > Previously, if caching was disabled due to Expires in the past, nginx > failed to cache the response even if it was cacheable as per subsequently > parsed Cache-Control header (ticket #964). > > Similarly, if caching was disabled due to Expires in the past, > "Cache-Control: no-cache" or "Cache-Control: max-age=0", caching was not > used if it was cacheable as per subsequently parsed X-Accel-Expires header. > > Fix is to avoid disabling caching immediately after parsing Expires in > the past or Cache-Control, but rather set flags which are later checked by > ngx_http_upstream_process_headers() (and cleared by "Cache-Control: max-age" > and X-Accel-Expires). > > Additionally, now X-Accel-Expires does not prevent parsing of cache control > extensions, notably stale-while-revalidate and stale-if-error.  This > ensures that order of the X-Accel-Expires and Cache-Control headers is not > important. > > Prodded by Vadim Fedorenko and Yugo Horie. > > diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c > --- a/src/http/ngx_http_upstream.c > +++ b/src/http/ngx_http_upstream.c > @@ -2697,6 +2697,10 @@ ngx_http_upstream_intercept_errors(ngx_h > >              if (r->cache) { > > +                if (u->headers_in.no_cache || u->headers_in.expired) { > +                    u->cacheable = 0; > +                } > + >                  if (u->cacheable) { >                      time_t  valid; > > @@ -2791,6 +2795,10 @@ ngx_http_upstream_process_headers(ngx_ht > >      umcf = ngx_http_get_module_main_conf(r, ngx_http_upstream_module); > > +    if (u->headers_in.no_cache || u->headers_in.expired) { > +        u->cacheable = 0; > +    } > + >      if (u->headers_in.x_accel_redirect >          && !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_REDIRECT)) >      { > @@ -4735,18 +4743,18 @@ ngx_http_upstream_process_cache_control( >          return NGX_OK; >      } > > -    if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { > -        return NGX_OK; > -    } > - >      start = h->value.data; >      last = start + h->value.len; > > +    if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { > +        goto extensions; > +    } > + >      if (ngx_strlcasestrn(start, last, (u_char *) "no-cache", 8 - 1) != NULL >          || ngx_strlcasestrn(start, last, (u_char *) "no-store", 8 - 1) != NULL >          || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) >      { > -        u->cacheable = 0; > +        u->headers_in.no_cache = 1; >          return NGX_OK; >      } > > @@ -4776,12 +4784,15 @@ ngx_http_upstream_process_cache_control( >          } > >          if (n == 0) { > -            u->cacheable = 0; > +            u->headers_in.no_cache = 1; >              return NGX_OK; >          } > >          r->cache->valid_sec = ngx_time() + n; > -    } > +        u->headers_in.expired = 0; > +    } > + > +extensions: > >      p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", >                           23 - 1); > @@ -4863,7 +4874,7 @@ ngx_http_upstream_process_expires(ngx_ht >      expires = ngx_parse_http_time(h->value.data, h->value.len); > >      if (expires == NGX_ERROR || expires < ngx_time()) { > -        u->cacheable = 0; > +        u->headers_in.expired = 1; >          return NGX_OK; >      } > > @@ -4914,6 +4925,8 @@ ngx_http_upstream_process_accel_expires( > >          default: >              r->cache->valid_sec = ngx_time() + n; > +            u->headers_in.no_cache = 0; > +            u->headers_in.expired = 0; >              return NGX_OK; >          } >      } > @@ -4925,6 +4938,8 @@ ngx_http_upstream_process_accel_expires( > >      if (n != NGX_ERROR) { >          r->cache->valid_sec = n; > +        u->headers_in.no_cache = 0; > +        u->headers_in.expired = 0; >      } >      } >  #endif > diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h > --- a/src/http/ngx_http_upstream.h > +++ b/src/http/ngx_http_upstream.h > @@ -297,6 +297,8 @@ typedef struct { > >      unsigned                         connection_close:1; >      unsigned                         chunked:1; > +    unsigned                         no_cache:1; > +    unsigned                         expired:1; >  } ngx_http_upstream_headers_in_t; > > > > -- > Maxim Dounin > http://mdounin.ru/ > _______________________________________________ > nginx-devel mailing list -- nginx-devel at nginx.org > To unsubscribe send an email to nginx-devel-leave at nginx.org > > From vadim.fedorenko at cdnnow.ru Mon Apr 25 21:04:53 2022 From: vadim.fedorenko at cdnnow.ru (Vadim Fedorenko) Date: Mon, 25 Apr 2022 22:04:53 +0100 Subject: [PATCH] Upstream: prioritise Cache-Control over Expires In-Reply-To: References: <4de897b7-e1b2-dbcd-7e20-f18ea91f8463@cdnnow.ru> <151ef936-f5d2-b681-bb54-6476c654a3ae@cdnnow.ru> <00218a02-f3e7-d1fe-04ba-1e61049f17be@cdnnow.ru> Message-ID: Hi Maxim! Thanks for the update, I will make some production tests soon but for now I can say that tests from test-suit are passing which is great! Best wishes, Vadim On 24.04.2022 16:42, Maxim Dounin wrote: > Hello! > > On Sun, Apr 24, 2022 at 07:55:17AM +0300, Maxim Dounin wrote: > > [...] > >> As far as I can tell, proper behaviour, assuming we parse cache >> control extensions independently of X-Accel-Expires, can be >> implemented by using just one flag. > > No, that's wrong, as with just one flag it wouldn't be possible to > correctly disable caching of responses with: > > Cache-Control: private > Cache-Control: max-age=10 > > So it needs at least two flags. Updated patch below, review and > testing appreciated. > > # HG changeset patch > # User Maxim Dounin > # Date 1650814681 -10800 > # Sun Apr 24 18:38:01 2022 +0300 > # Node ID 940ba4317a97c72d1ee6700cbf58a543fee04c7a > # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b > Upstream: fixed X-Accel-Expires/Cache-Control/Expires handling. > > Previously, if caching was disabled due to Expires in the past, nginx > failed to cache the response even if it was cacheable as per subsequently > parsed Cache-Control header (ticket #964). > > Similarly, if caching was disabled due to Expires in the past, > "Cache-Control: no-cache" or "Cache-Control: max-age=0", caching was not > used if it was cacheable as per subsequently parsed X-Accel-Expires header. > > Fix is to avoid disabling caching immediately after parsing Expires in > the past or Cache-Control, but rather set flags which are later checked by > ngx_http_upstream_process_headers() (and cleared by "Cache-Control: max-age" > and X-Accel-Expires). > > Additionally, now X-Accel-Expires does not prevent parsing of cache control > extensions, notably stale-while-revalidate and stale-if-error. This > ensures that order of the X-Accel-Expires and Cache-Control headers is not > important. > > Prodded by Vadim Fedorenko and Yugo Horie. > > diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c > --- a/src/http/ngx_http_upstream.c > +++ b/src/http/ngx_http_upstream.c > @@ -2697,6 +2697,10 @@ ngx_http_upstream_intercept_errors(ngx_h > > if (r->cache) { > > + if (u->headers_in.no_cache || u->headers_in.expired) { > + u->cacheable = 0; > + } > + > if (u->cacheable) { > time_t valid; > > @@ -2791,6 +2795,10 @@ ngx_http_upstream_process_headers(ngx_ht > > umcf = ngx_http_get_module_main_conf(r, ngx_http_upstream_module); > > + if (u->headers_in.no_cache || u->headers_in.expired) { > + u->cacheable = 0; > + } > + > if (u->headers_in.x_accel_redirect > && !(u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_XA_REDIRECT)) > { > @@ -4735,18 +4743,18 @@ ngx_http_upstream_process_cache_control( > return NGX_OK; > } > > - if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { > - return NGX_OK; > - } > - > start = h->value.data; > last = start + h->value.len; > > + if (r->cache->valid_sec != 0 && u->headers_in.x_accel_expires != NULL) { > + goto extensions; > + } > + > if (ngx_strlcasestrn(start, last, (u_char *) "no-cache", 8 - 1) != NULL > || ngx_strlcasestrn(start, last, (u_char *) "no-store", 8 - 1) != NULL > || ngx_strlcasestrn(start, last, (u_char *) "private", 7 - 1) != NULL) > { > - u->cacheable = 0; > + u->headers_in.no_cache = 1; > return NGX_OK; > } > > @@ -4776,12 +4784,15 @@ ngx_http_upstream_process_cache_control( > } > > if (n == 0) { > - u->cacheable = 0; > + u->headers_in.no_cache = 1; > return NGX_OK; > } > > r->cache->valid_sec = ngx_time() + n; > - } > + u->headers_in.expired = 0; > + } > + > +extensions: > > p = ngx_strlcasestrn(start, last, (u_char *) "stale-while-revalidate=", > 23 - 1); > @@ -4863,7 +4874,7 @@ ngx_http_upstream_process_expires(ngx_ht > expires = ngx_parse_http_time(h->value.data, h->value.len); > > if (expires == NGX_ERROR || expires < ngx_time()) { > - u->cacheable = 0; > + u->headers_in.expired = 1; > return NGX_OK; > } > > @@ -4914,6 +4925,8 @@ ngx_http_upstream_process_accel_expires( > > default: > r->cache->valid_sec = ngx_time() + n; > + u->headers_in.no_cache = 0; > + u->headers_in.expired = 0; > return NGX_OK; > } > } > @@ -4925,6 +4938,8 @@ ngx_http_upstream_process_accel_expires( > > if (n != NGX_ERROR) { > r->cache->valid_sec = n; > + u->headers_in.no_cache = 0; > + u->headers_in.expired = 0; > } > } > #endif > diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h > --- a/src/http/ngx_http_upstream.h > +++ b/src/http/ngx_http_upstream.h > @@ -297,6 +297,8 @@ typedef struct { > > unsigned connection_close:1; > unsigned chunked:1; > + unsigned no_cache:1; > + unsigned expired:1; > } ngx_http_upstream_headers_in_t; > > > From yugo-horie at jocdn.co.jp Tue Apr 26 00:18:41 2022 From: yugo-horie at jocdn.co.jp (Yugo Horie) Date: Tue, 26 Apr 2022 09:18:41 +0900 Subject: [PATCH v2] Tests: added Expires and Cache-Control headers test In-Reply-To: <3d5684530a8ef228cd7f.1650406842@repo.dev.cdnnow.net> References: <3d5684530a8ef228cd7f.1650406842@repo.dev.cdnnow.net> Message-ID: Hi, Vadim We assume that your test has been passed whether applying Maxim's patch or not the conventional one. (my verification may be inadequate...) : server { : listen 127.0.0.1:8080; : location / { : proxy_pass http://127.0.0.2:8080; : proxy_cache cache_zone; : } : } : server { : listen 127.0.0.2:8080; When I changed the upstream server port to another one(e.g 8081), The conventional Nginx had failed on the `/fail` case and Maxim's patched ones had come to pass. We consider that this seems to be the right direction. And I prefer to add the cache status headers instead of using your cache file counter. The config phase: : server { : listen 127.0.0.1:8080; : location / { : proxy_pass http://127.0.0.2:8081; : proxy_cache cache_zone; : add_header X-Cache-Status \$upstream_cache_status; : } : } : server { : listen 127.0.0.2:8081; : location /success { : add_header Cache-Control "max-age=3600"; : add_header Expires "Tue, 15 Nov 1994 12:45:26 GMT"; : return 200 "Hello world!"; : } : location /fail { : add_header Expires "Tue, 15 Nov 1994 12:45:26 GMT"; : add_header Cache-Control "max-age=3600"; : return 200 "Hello world!"; : } On the test phase: : $r = http_get('/success'); : like($r, qr/Cache-Control/, 'cache-control-first-cache-control-is-present'); : like($r, qr/Expires/, 'cache-control-first-expires-is-present'); : $r = http_get('/success'); : like($r, qr/X-Cache-Status: HIT/, 'cache-status-is-hit'); : : $r = http_get('/fail'); : like($r, qr/Cache-Control/, 'expires-first-cache-control-is-present'); : like($r, qr/Expires/, 'expires-first-expires-is-present'); : $r = http_get('/fail'); : like($r, qr/X-Cache-Status: HIT/, 'cache-status-is-hit'); Thanks a lot, Yugo Horie 2022年4月20日(水) 7:22 Vadim Fedorenko via nginx-devel : > > # HG changeset patch > # User Vadim Fedorenko > # Date 1649976970 -10800 > # Fri Apr 15 01:56:10 2022 +0300 > # Node ID 3d5684530a8ef228cd7f20ff3e51f9ea5e77f2ec > # Parent 0c50a00e67334659d58d3cf7cb81fcf5872a8285 > Tests: added Expires and Cache-Control headers test > > diff -r 0c50a00e6733 -r 3d5684530a8e proxy_cache_expires_cache_control.t > --- /dev/null Thu Jan 01 00:00:00 1970 +0000 > +++ b/proxy_cache_expires_cache_control.t Fri Apr 15 01:56:10 2022 +0300 > @@ -0,0 +1,99 @@ > +#!/usr/bin/perl > + > +# (C) Georgii Dzebisashvili > +# (C) Vadim Fedorenko > + > +# Tests for cache management regarding https://trac.nginx.org/nginx/ticket/964 > + > +############################################################################### > + > +use warnings; > +use strict; > + > +use Test::More; > + > +BEGIN { use FindBin; chdir($FindBin::Bin); } > + > +use lib 'lib'; > +use Test::Nginx; > + > +use File::Find; > + > +############################################################################### > + > +select STDERR; $| = 1; > +select STDOUT; $| = 1; > + > +my $scriptDir = $FindBin::Bin; > +my $cacheDir = $scriptDir."/testcache"; > +my $tempDir = $scriptDir."/testtemp"; > + > +my $t = Test::Nginx->new()->plan(8)->write_file_expand('nginx.conf', <<"EOF"); > + > +daemon off; > +events { > + worker_connections 64; > +} > +http { > + access_log off; > + proxy_cache_path $cacheDir levels=2 keys_zone=cache_zone:1m inactive=24h max_size=10m; > + proxy_temp_path $tempDir; > + server { > + listen 127.0.0.1:8080; > + location / { > + proxy_pass http://127.0.0.2:8080; > + proxy_cache cache_zone; > + } > + } > + server { > + listen 127.0.0.2:8080; > + location /success { > + add_header Cache-Control "max-age=3600"; > + add_header Expires "Tue, 15 Nov 1994 12:45:26 GMT"; > + return 200 "Hello world!"; > + } > + location /fail { > + add_header Expires "Tue, 15 Nov 1994 12:45:26 GMT"; > + add_header Cache-Control "max-age=3600"; > + return 200 "Hello world!"; > + } > + location /store_xaccelexpire { > + add_header Expires "Tue, 15 Nov 1994 12:45:26 GMT"; > + add_header Cache-Control "no-cache"; > + add_header X-Accel-Expires "60"; > + return 200 "Hello world!"; > + } > + } > +} > + > + > +EOF > + > +$t->run(); > + > +############################################################################### > + > +my $counter = 0; > + > +sub fileMatchCallback { > + -f && $counter++; # Only count files > +} > + > +my $r; > + > +$r = http_get('/success'); > +like($r, qr/Cache-Control/, 'cache-control-first-cache-control-is-present'); > +like($r, qr/Expires/, 'cache-control-first-expires-is-present'); > + > +$r = http_get('/fail'); > +like($r, qr/Cache-Control/, 'expires-first-cache-control-is-present'); > +like($r, qr/Expires/, 'expires-first-expires-is-present'); > + > +$r = http_get('/store_xaccelexpire'); > +like($r, qr/Cache-Control/, 'cache-control-with-xaccel-is-present'); > +like($r, qr/Expires/, 'expires-with-xaccel-is-present'); > +unlike($r, qr/X-Accel-Expires/, 'xaccel-is-not-present'); > + > +find(\&fileMatchCallback, $cacheDir); > + > +is($counter, 3, 'overall number of cached requests'); > _______________________________________________ > nginx-devel mailing list -- nginx-devel at nginx.org > To unsubscribe send an email to nginx-devel-leave at nginx.org From xeioex at nginx.com Tue Apr 26 23:09:00 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 26 Apr 2022 23:09:00 +0000 Subject: [njs] Fixed aggregation methods of Promise ctor with array-like object. Message-ID: details: https://hg.nginx.org/njs/rev/744122b4dccf branches: changeset: 1841:744122b4dccf user: Dmitry Volyntsev date: Tue Apr 26 16:07:02 2022 -0700 description: Fixed aggregation methods of Promise ctor with array-like object. Previously, while iterating over an array-like object the methods may be resolved with INVALID values. INVALID value is a special internal type which should never be visible by ordinary functions. The fix is to ensure that absent elements are represented by undefined value. The following methods were fixed Promise.all(), Promise.allSettled(), Promise.any(), Promise.race(). This closes #483 issue on Github. diffstat: src/njs_promise.c | 16 ++++++++++++++++ src/test/njs_unit_test.c | 7 +++++++ 2 files changed, 23 insertions(+), 0 deletions(-) diffs (64 lines): diff -r 3fec53d722ef -r 744122b4dccf src/njs_promise.c --- a/src/njs_promise.c Fri Apr 22 17:02:36 2022 -0700 +++ b/src/njs_promise.c Tue Apr 26 16:07:02 2022 -0700 @@ -1369,6 +1369,10 @@ njs_promise_perform_all_handler(njs_vm_t njs_promise_all_context_t *context; njs_promise_iterator_args_t *pargs; + if (!njs_is_valid(value)) { + value = njs_value_arg(&njs_value_undefined); + } + pargs = (njs_promise_iterator_args_t *) args; capability = pargs->capability; @@ -1459,6 +1463,10 @@ njs_promise_perform_all_settled_handler( njs_promise_all_context_t *context; njs_promise_iterator_args_t *pargs; + if (!njs_is_valid(value)) { + value = njs_value_arg(&njs_value_undefined); + } + pargs = (njs_promise_iterator_args_t *) args; capability = pargs->capability; @@ -1598,6 +1606,10 @@ njs_promise_perform_any_handler(njs_vm_t njs_promise_all_context_t *context; njs_promise_iterator_args_t *pargs; + if (!njs_is_valid(value)) { + value = njs_value_arg(&njs_value_undefined); + } + pargs = (njs_promise_iterator_args_t *) args; capability = pargs->capability; @@ -1745,6 +1757,10 @@ njs_promise_perform_race_handler(njs_vm_ njs_promise_capability_t *capability; njs_promise_iterator_args_t *pargs; + if (!njs_is_valid(value)) { + value = njs_value_arg(&njs_value_undefined); + } + pargs = (njs_promise_iterator_args_t *) args; ret = njs_function_call(vm, pargs->function, pargs->constructor, value, diff -r 3fec53d722ef -r 744122b4dccf src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Fri Apr 22 17:02:36 2022 -0700 +++ b/src/test/njs_unit_test.c Tue Apr 26 16:07:02 2022 -0700 @@ -21149,6 +21149,13 @@ static njs_unit_test_t njs_externals_te "}))" ".then(v => $r.retval(v))"), njs_str("a:async:pr:async2:pr:r,b:async:pr:async2:pr:r,c:async:pr:async2:pr:r") }, + + { njs_str("async function f () {" + " var p = await Promise.race({length:1});" + " for (const v in 'test') { }" + "};" + "f().then(v => $r.retval('done'))"), + njs_str("done") }, }; From xeioex at nginx.com Tue Apr 26 23:09:02 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 26 Apr 2022 23:09:02 +0000 Subject: [njs] Making FOREACH and NEXT instructions robust to invalid input. Message-ID: details: https://hg.nginx.org/njs/rev/f150bd2d40bb branches: changeset: 1842:f150bd2d40bb user: Dmitry Volyntsev date: Tue Apr 26 16:07:06 2022 -0700 description: Making FOREACH and NEXT instructions robust to invalid input. diffstat: src/njs_value.h | 3 +-- src/njs_vmcode.c | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diffs (42 lines): diff -r 744122b4dccf -r f150bd2d40bb src/njs_value.h --- a/src/njs_value.h Tue Apr 26 16:07:02 2022 -0700 +++ b/src/njs_value.h Tue Apr 26 16:07:06 2022 -0700 @@ -70,7 +70,7 @@ typedef enum { NJS_DATA_TAG_TEXT_ENCODER, NJS_DATA_TAG_TEXT_DECODER, NJS_DATA_TAG_ARRAY_ITERATOR, - NJS_DATA_TAG_FS_STAT, + NJS_DATA_TAG_FOREACH_NEXT, NJS_DATA_TAG_MAX } njs_data_tag_t; @@ -135,7 +135,6 @@ union njs_value_s { njs_promise_t *promise; njs_prop_handler_t prop_handler; njs_value_t *value; - njs_property_next_t *next; void *data; } u; } data; diff -r 744122b4dccf -r f150bd2d40bb src/njs_vmcode.c --- a/src/njs_vmcode.c Tue Apr 26 16:07:02 2022 -0700 +++ b/src/njs_vmcode.c Tue Apr 26 16:07:06 2022 -0700 @@ -819,7 +819,8 @@ next: pnext = (njs_vmcode_prop_next_t *) pc; retval = njs_scope_value(vm, pnext->retval); - next = value2->data.u.next; + njs_assert(njs_is_data(value2, NJS_DATA_TAG_FOREACH_NEXT)); + next = njs_data(value2); if (next->index < next->array->length) { *retval = next->array->start[next->index++]; @@ -1468,7 +1469,7 @@ njs_vmcode_property_foreach(njs_vm_t *vm return NJS_ERROR; } - vm->retval.data.u.next = next; + njs_set_data(&vm->retval, next, NJS_DATA_TAG_FOREACH_NEXT); code = (njs_vmcode_prop_foreach_t *) pc; From mdounin at mdounin.ru Wed Apr 27 23:55:24 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 28 Apr 2022 02:55:24 +0300 Subject: patch: call modules' exit_process in reverse order In-Reply-To: <564de12e05a6d3b229e395855712255f@asiainfo.com> References: <564de12e05a6d3b229e395855712255f@asiainfo.com> Message-ID: Hello! On Sun, Apr 24, 2022 at 04:33:04PM +0800, shanlei at asiainfo.com wrote: > # HG changeset patch > # User stdanley > # Date 1650788278 -28800 > # Sun Apr 24 16:17:58 2022 +0800 > # Node ID 522acbe88486d027383075c8208edd6fcc0a3aa6 > # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b > patch: call modules' exit_process in reverse order to solve module > dependency. > We once have developed a module which depends on ngx_thread_pool module. > but nginx hungs when began to shutdown. trace shows that ngx_thread_pool > cant finish "exit_process" because our module is using a thread, our > module must > call "exit_process" before ngx_thread_pool. > so we present this patch to solve modules' dependency issue. Thank you for the patch. First of all, you may want to consider this section in the development guide: http://nginx.org/en/docs/dev/development_guide.html#threads_pitfalls Using threads in nginx is usually a bad idea. Thread pools, in particular, are designed to be used to offload short operations like read() syscalls. And these operations are expected to be finished before nginx exit, so thread pool's exit handler waits for all threads to complete. If you are sure you have to use threads, and thread pools in particular, and with some long-running tasks, you may want to consider other ways to signal your threads to exit. In particular, checking ngx_exiting/ngx_quit/ngx_terminate periodically might be a way to go. Though I would recommend to reconsider using long-running thread tasks anyway. As for the patch, it looks wrong to me. While changing the exit_process() callback order might solve your particular dependency issue, similar issues might easily appear with any order. Hope this helps. -- Maxim Dounin http://mdounin.ru/ From xeioex at nginx.com Thu Apr 28 00:06:59 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 28 Apr 2022 00:06:59 +0000 Subject: [njs] Fixed Array.prototype.lastIndexOf() with unicode string as "this". Message-ID: details: https://hg.nginx.org/njs/rev/251e7ab400a8 branches: changeset: 1843:251e7ab400a8 user: Dmitry Volyntsev date: Wed Apr 27 16:31:00 2022 -0700 description: Fixed Array.prototype.lastIndexOf() with unicode string as "this". Previously, when lastIndexOf() was called with unicode string as "this" argument and a negative "fromIndex" argument null-pointer dererence might occur because njs_string_offset() was called with invalid index value whereas njs_string_offset() should always be called with valid index argument. The fix is to verify that from index is valid. This closes #482 issue on Github. diffstat: src/njs_iterator.c | 9 ++++++--- src/test/njs_unit_test.c | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diffs (33 lines): diff -r f150bd2d40bb -r 251e7ab400a8 src/njs_iterator.c --- a/src/njs_iterator.c Tue Apr 26 16:07:06 2022 -0700 +++ b/src/njs_iterator.c Wed Apr 27 16:31:00 2022 -0700 @@ -560,10 +560,13 @@ njs_object_iterate_reverse(njs_vm_t *vm, } else { /* UTF-8 string. */ - p = njs_string_offset(string_prop.start, end, from); - p = njs_utf8_next(p, end); + p = NULL; + i = from + 1; - i = from + 1; + if (i > to) { + p = njs_string_offset(string_prop.start, end, from); + p = njs_utf8_next(p, end); + } while (i-- > to) { pos = njs_utf8_prev(p); diff -r f150bd2d40bb -r 251e7ab400a8 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Tue Apr 26 16:07:06 2022 -0700 +++ b/src/test/njs_unit_test.c Wed Apr 27 16:31:00 2022 -0700 @@ -5103,6 +5103,9 @@ static njs_unit_test_t njs_test[] = { njs_str("Array.prototype.lastIndexOf.call({0:'undefined', length:0}, 'undefined')"), njs_str("-1") }, + { njs_str("[1,0,-1,-2].map(v => Array.prototype.lastIndexOf.call('Ф', 'Ф', v))"), + njs_str("0,0,0,-1") }, + { njs_str("[''].lastIndexOf.call('00000000000000000000000000000а00')"), njs_str("-1") }, From georg at syscid.com Thu Apr 28 11:37:28 2022 From: georg at syscid.com (Georg Pfuetzenreuter) Date: Thu, 28 Apr 2022 13:37:28 +0200 Subject: [PATCH] Clarify 495 response message Message-ID: Hello! The current message "The SSL certificate error" feels cut off and is ambiguous. I suggest the following change, which clarifies the 495 response whilst keeping it flexible enough to be used for different client certificate errors. Best, Georg # HG changeset patch # User Georg Pfuetzenreuter # Date 1651145208 -7200 # Thu Apr 28 13:26:48 2022 +0200 # Node ID 9c2491a6c4615647768a13280eaa1e22b20fc36d # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b Clarify 495 response Adjust HTTP 495 response message shown to an end user to make more sense. diff -r a736a7a613ea -r 9c2491a6c461 src/http/ngx_http_special_response.c --- a/src/http/ngx_http_special_response.c Tue Feb 08 17:35:27 2022 +0300 +++ b/src/http/ngx_http_special_response.c Thu Apr 28 13:26:48 2022 +0200 @@ -253,11 +253,11 @@ static char ngx_http_error_495_page[] = "" CRLF -"400 The SSL certificate error" +"400 SSL client certificate error" CRLF "" CRLF "

400 Bad Request

" CRLF -"
The SSL certificate error
" CRLF +"
SSL client certificate error
" CRLF ; -------------- next part -------------- A non-text attachment was scrubbed... Name: OpenPGP_0x3DAFF26C3E79E588.asc Type: application/pgp-keys Size: 660 bytes Desc: OpenPGP public key URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: OpenPGP_signature Type: application/pgp-signature Size: 236 bytes Desc: OpenPGP digital signature URL: From xeioex at nginx.com Fri Apr 29 00:18:24 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 29 Apr 2022 00:18:24 +0000 Subject: [njs] Modules: added additional directives for Fetch API. Message-ID: details: https://hg.nginx.org/njs/rev/b20de7bcee61 branches: changeset: 1844:b20de7bcee61 user: Dmitry Volyntsev date: Thu Apr 28 16:37:14 2022 -0700 description: Modules: added additional directives for Fetch API. The following directives are added: * js_fetch_timeout * js_fetch_verify * js_fetch_buffer_size * js_fetch_max_response_buffer_size This closes #489 issue on Github. diffstat: nginx/ngx_http_js_module.c | 105 ++++++++++++++++++++++++++++++++++++++++++- nginx/ngx_js.h | 16 +++++- nginx/ngx_js_fetch.c | 8 ++- nginx/ngx_stream_js_module.c | 106 ++++++++++++++++++++++++++++++++++++++++++- ts/ngx_core.d.ts | 6 +- 5 files changed, 232 insertions(+), 9 deletions(-) diffs (470 lines): diff -r 251e7ab400a8 -r b20de7bcee61 nginx/ngx_http_js_module.c --- a/nginx/ngx_http_js_module.c Wed Apr 27 16:31:00 2022 -0700 +++ b/nginx/ngx_http_js_module.c Thu Apr 28 16:37:14 2022 -0700 @@ -24,10 +24,16 @@ typedef struct { ngx_str_t header_filter; ngx_str_t body_filter; ngx_uint_t buffer_type; + + size_t buffer_size; + size_t max_response_body_size; + ngx_msec_t timeout; + #if (NGX_HTTP_SSL) ngx_ssl_t *ssl; ngx_str_t ssl_ciphers; ngx_uint_t ssl_protocols; + ngx_flag_t ssl_verify; ngx_int_t ssl_verify_depth; ngx_str_t ssl_trusted_certificate; #endif @@ -207,6 +213,11 @@ static ngx_resolver_t *ngx_http_js_resol ngx_http_request_t *r); static ngx_msec_t ngx_http_js_resolver_timeout(njs_vm_t *vm, ngx_http_request_t *r); +static ngx_msec_t ngx_http_js_fetch_timeout(njs_vm_t *vm, + ngx_http_request_t *r); +static size_t ngx_http_js_buffer_size(njs_vm_t *vm, ngx_http_request_t *r); +static size_t ngx_http_js_max_response_buffer_size(njs_vm_t *vm, + ngx_http_request_t *r); static void ngx_http_js_handle_vm_event(ngx_http_request_t *r, njs_vm_event_t vm_event, njs_value_t *args, njs_uint_t nargs); static void ngx_http_js_handle_event(ngx_http_request_t *r, @@ -231,6 +242,7 @@ static char *ngx_http_js_merge_loc_conf( static char * ngx_http_js_set_ssl(ngx_conf_t *cf, ngx_http_js_loc_conf_t *jlcf); #endif static ngx_ssl_t *ngx_http_js_ssl(njs_vm_t *vm, ngx_http_request_t *r); +static ngx_flag_t ngx_http_js_ssl_verify(njs_vm_t *vm, ngx_http_request_t *r); #if (NGX_HTTP_SSL) @@ -295,6 +307,27 @@ static ngx_command_t ngx_http_js_comman 0, NULL }, + { ngx_string("js_fetch_buffer_size"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_js_loc_conf_t, buffer_size), + NULL }, + + { ngx_string("js_fetch_max_response_buffer_size"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_js_loc_conf_t, max_response_body_size), + NULL }, + + { ngx_string("js_fetch_timeout"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_js_loc_conf_t, timeout), + NULL }, + #if (NGX_HTTP_SSL) { ngx_string("js_fetch_ciphers"), @@ -311,6 +344,13 @@ static ngx_command_t ngx_http_js_comman offsetof(ngx_http_js_loc_conf_t, ssl_protocols), &ngx_http_js_ssl_protocols }, + { ngx_string("js_fetch_verify"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_js_loc_conf_t, ssl_verify), + NULL }, + { ngx_string("js_fetch_verify_depth"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, @@ -716,11 +756,15 @@ static uintptr_t ngx_http_js_uptr[] = { (uintptr_t) ngx_http_js_resolver_timeout, (uintptr_t) ngx_http_js_handle_event, (uintptr_t) ngx_http_js_ssl, + (uintptr_t) ngx_http_js_ssl_verify, + (uintptr_t) ngx_http_js_fetch_timeout, + (uintptr_t) ngx_http_js_buffer_size, + (uintptr_t) ngx_http_js_max_response_buffer_size, }; static njs_vm_meta_t ngx_http_js_metas = { - .size = 6, + .size = njs_nitems(ngx_http_js_uptr), .values = ngx_http_js_uptr }; @@ -3420,6 +3464,39 @@ ngx_http_js_resolver_timeout(njs_vm_t *v } +static ngx_msec_t +ngx_http_js_fetch_timeout(njs_vm_t *vm, ngx_http_request_t *r) +{ + ngx_http_js_loc_conf_t *jlcf; + + jlcf = ngx_http_get_module_loc_conf(r, ngx_http_js_module); + + return jlcf->timeout; +} + + +static size_t +ngx_http_js_buffer_size(njs_vm_t *vm, ngx_http_request_t *r) +{ + ngx_http_js_loc_conf_t *jlcf; + + jlcf = ngx_http_get_module_loc_conf(r, ngx_http_js_module); + + return jlcf->buffer_size; +} + + +static size_t +ngx_http_js_max_response_buffer_size(njs_vm_t *vm, ngx_http_request_t *r) +{ + ngx_http_js_loc_conf_t *jlcf; + + jlcf = ngx_http_get_module_loc_conf(r, ngx_http_js_module); + + return jlcf->max_response_body_size; +} + + static void ngx_http_js_handle_vm_event(ngx_http_request_t *r, njs_vm_event_t vm_event, njs_value_t *args, njs_uint_t nargs) @@ -3952,7 +4029,12 @@ ngx_http_js_create_loc_conf(ngx_conf_t * * conf->ssl_trusted_certificate = { 0, NULL }; */ + conf->buffer_size = NGX_CONF_UNSET_SIZE; + conf->max_response_body_size = NGX_CONF_UNSET_SIZE; + conf->timeout = NGX_CONF_UNSET_MSEC; + #if (NGX_HTTP_SSL) + conf->ssl_verify = NGX_CONF_UNSET; conf->ssl_verify_depth = NGX_CONF_UNSET; #endif @@ -3972,6 +4054,11 @@ ngx_http_js_merge_loc_conf(ngx_conf_t *c ngx_conf_merge_uint_value(conf->buffer_type, prev->buffer_type, NGX_JS_STRING); + ngx_conf_merge_msec_value(conf->timeout, prev->timeout, 60000); + ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size, 16384); + ngx_conf_merge_size_value(conf->max_response_body_size, + prev->max_response_body_size, 1048576); + #if (NGX_HTTP_SSL) ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); @@ -3979,6 +4066,7 @@ ngx_http_js_merge_loc_conf(ngx_conf_t *c (NGX_CONF_BITMASK_SET|NGX_SSL_TLSv1 |NGX_SSL_TLSv1_1|NGX_SSL_TLSv1_2)); + ngx_conf_merge_value(conf->ssl_verify, prev->ssl_verify, 1); ngx_conf_merge_value(conf->ssl_verify_depth, prev->ssl_verify_depth, 100); ngx_conf_merge_str_value(conf->ssl_trusted_certificate, @@ -4050,3 +4138,18 @@ ngx_http_js_ssl(njs_vm_t *vm, ngx_http_r return NULL; #endif } + + +static ngx_flag_t +ngx_http_js_ssl_verify(njs_vm_t *vm, ngx_http_request_t *r) +{ +#if (NGX_HTTP_SSL) + ngx_http_js_loc_conf_t *jlcf; + + jlcf = ngx_http_get_module_loc_conf(r, ngx_http_js_module); + + return jlcf->ssl_verify; +#else + return 0; +#endif +} diff -r 251e7ab400a8 -r b20de7bcee61 nginx/ngx_js.h --- a/nginx/ngx_js.h Wed Apr 27 16:31:00 2022 -0700 +++ b/nginx/ngx_js.h Thu Apr 28 16:37:14 2022 -0700 @@ -25,7 +25,11 @@ typedef void (*ngx_js_event_handler_pt)( njs_vm_event_t vm_event, njs_value_t *args, njs_uint_t nargs); typedef ngx_resolver_t *(*ngx_external_resolver_pt)(njs_vm_t *vm, njs_external_ptr_t e); -typedef ngx_msec_t (*ngx_external_resolver_timeout_pt)(njs_vm_t *vm, +typedef ngx_msec_t (*ngx_external_timeout_pt)(njs_vm_t *vm, + njs_external_ptr_t e); +typedef ngx_flag_t (*ngx_external_flag_pt)(njs_vm_t *vm, + njs_external_ptr_t e); +typedef ngx_flag_t (*ngx_external_size_pt)(njs_vm_t *vm, njs_external_ptr_t e); typedef ngx_ssl_t *(*ngx_external_ssl_pt)(njs_vm_t *vm, njs_external_ptr_t e); @@ -37,11 +41,19 @@ typedef ngx_ssl_t *(*ngx_external_ssl_pt #define ngx_external_resolver(vm, e) \ ((ngx_external_resolver_pt) njs_vm_meta(vm, 2))(vm, e) #define ngx_external_resolver_timeout(vm, e) \ - ((ngx_external_resolver_timeout_pt) njs_vm_meta(vm, 3))(vm, e) + ((ngx_external_timeout_pt) njs_vm_meta(vm, 3))(vm, e) #define ngx_external_event_handler(vm, e) \ ((ngx_js_event_handler_pt) njs_vm_meta(vm, 4)) #define ngx_external_ssl(vm, e) \ ((ngx_external_ssl_pt) njs_vm_meta(vm, 5))(vm, e) +#define ngx_external_ssl_verify(vm, e) \ + ((ngx_external_flag_pt) njs_vm_meta(vm, 6))(vm, e) +#define ngx_external_fetch_timeout(vm, e) \ + ((ngx_external_timeout_pt) njs_vm_meta(vm, 7))(vm, e) +#define ngx_external_buffer_size(vm, e) \ + ((ngx_external_size_pt) njs_vm_meta(vm, 8))(vm, e) +#define ngx_external_max_response_buffer_size(vm, e) \ + ((ngx_external_size_pt) njs_vm_meta(vm, 9))(vm, e) #define ngx_js_prop(vm, type, value, start, len) \ diff -r 251e7ab400a8 -r b20de7bcee61 nginx/ngx_js_fetch.c --- a/nginx/ngx_js_fetch.c Wed Apr 27 16:31:00 2022 -0700 +++ b/nginx/ngx_js_fetch.c Thu Apr 28 16:37:14 2022 -0700 @@ -376,9 +376,11 @@ ngx_js_ext_fetch(njs_vm_t *vm, njs_value } http->external = external; + http->timeout = ngx_external_fetch_timeout(vm, external); http->event_handler = ngx_external_event_handler(vm, external); - http->buffer_size = 4096; - http->max_response_body_size = 32 * 1024; + http->buffer_size = ngx_external_buffer_size(vm, external); + http->max_response_body_size = + ngx_external_max_response_buffer_size(vm, external); ret = ngx_js_string(vm, njs_arg(args, nargs, 1), &http->url); if (ret != NJS_OK) { @@ -408,7 +410,7 @@ ngx_js_ext_fetch(njs_vm_t *vm, njs_value u.url.data += 8; u.default_port = 443; http->ssl = ngx_external_ssl(vm, external); - http->ssl_verify = 1; + http->ssl_verify = ngx_external_ssl_verify(vm, external); #endif } else { diff -r 251e7ab400a8 -r b20de7bcee61 nginx/ngx_stream_js_module.c --- a/nginx/ngx_stream_js_module.c Wed Apr 27 16:31:00 2022 -0700 +++ b/nginx/ngx_stream_js_module.c Thu Apr 28 16:37:14 2022 -0700 @@ -31,9 +31,15 @@ typedef struct { ngx_str_t access; ngx_str_t preread; ngx_str_t filter; + + size_t buffer_size; + size_t max_response_body_size; + ngx_msec_t timeout; + #if (NGX_STREAM_SSL) ngx_ssl_t *ssl; ngx_str_t ssl_ciphers; + ngx_flag_t ssl_verify; ngx_uint_t ssl_protocols; ngx_int_t ssl_verify_depth; ngx_str_t ssl_trusted_certificate; @@ -124,6 +130,11 @@ static ngx_resolver_t *ngx_stream_js_res ngx_stream_session_t *s); static ngx_msec_t ngx_stream_js_resolver_timeout(njs_vm_t *vm, ngx_stream_session_t *s); +static ngx_msec_t ngx_stream_js_fetch_timeout(njs_vm_t *vm, + ngx_stream_session_t *s); +static size_t ngx_stream_js_buffer_size(njs_vm_t *vm, ngx_stream_session_t *s); +static size_t ngx_stream_js_max_response_buffer_size(njs_vm_t *vm, + ngx_stream_session_t *s); static void ngx_stream_js_handle_event(ngx_stream_session_t *s, njs_vm_event_t vm_event, njs_value_t *args, njs_uint_t nargs); @@ -145,6 +156,8 @@ static char * ngx_stream_js_set_ssl(ngx_ ngx_stream_js_srv_conf_t *jscf); #endif static ngx_ssl_t *ngx_stream_js_ssl(njs_vm_t *vm, ngx_stream_session_t *s); +static ngx_flag_t ngx_stream_js_ssl_verify(njs_vm_t *vm, + ngx_stream_session_t *s); #if (NGX_STREAM_SSL) @@ -209,6 +222,27 @@ static ngx_command_t ngx_stream_js_comm offsetof(ngx_stream_js_srv_conf_t, filter), NULL }, + { ngx_string("js_fetch_buffer_size"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_js_srv_conf_t, buffer_size), + NULL }, + + { ngx_string("js_fetch_max_response_buffer_size"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_js_srv_conf_t, max_response_body_size), + NULL }, + + { ngx_string("js_fetch_timeout"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_js_srv_conf_t, timeout), + NULL }, + #if (NGX_STREAM_SSL) { ngx_string("js_fetch_ciphers"), @@ -225,6 +259,13 @@ static ngx_command_t ngx_stream_js_comm offsetof(ngx_stream_js_srv_conf_t, ssl_protocols), &ngx_stream_js_ssl_protocols }, + { ngx_string("js_fetch_verify"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_js_srv_conf_t, ssl_verify), + NULL }, + { ngx_string("js_fetch_verify_depth"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, @@ -468,11 +509,15 @@ static uintptr_t ngx_stream_js_uptr[] = (uintptr_t) ngx_stream_js_resolver_timeout, (uintptr_t) ngx_stream_js_handle_event, (uintptr_t) ngx_stream_js_ssl, + (uintptr_t) ngx_stream_js_ssl_verify, + (uintptr_t) ngx_stream_js_fetch_timeout, + (uintptr_t) ngx_stream_js_buffer_size, + (uintptr_t) ngx_stream_js_max_response_buffer_size, }; static njs_vm_meta_t ngx_stream_js_metas = { - .size = 6, + .size = njs_nitems(ngx_stream_js_uptr), .values = ngx_stream_js_uptr }; @@ -1456,6 +1501,39 @@ ngx_stream_js_resolver_timeout(njs_vm_t } +static ngx_msec_t +ngx_stream_js_fetch_timeout(njs_vm_t *vm, ngx_stream_session_t *s) +{ + ngx_stream_core_srv_conf_t *cscf; + + cscf = ngx_stream_get_module_srv_conf(s, ngx_stream_js_module); + + return cscf->resolver_timeout; +} + + +static size_t +ngx_stream_js_buffer_size(njs_vm_t *vm, ngx_stream_session_t *s) +{ + ngx_stream_js_srv_conf_t *jscf; + + jscf = ngx_stream_get_module_srv_conf(s, ngx_stream_js_module); + + return jscf->buffer_size; +} + + +static size_t +ngx_stream_js_max_response_buffer_size(njs_vm_t *vm, ngx_stream_session_t *s) +{ + ngx_stream_js_srv_conf_t *jscf; + + jscf = ngx_stream_get_module_srv_conf(s, ngx_stream_js_module); + + return jscf->max_response_body_size; +} + + static void ngx_stream_js_handle_event(ngx_stream_session_t *s, njs_vm_event_t vm_event, njs_value_t *args, njs_uint_t nargs) @@ -1898,7 +1976,12 @@ ngx_stream_js_create_srv_conf(ngx_conf_t * conf->ssl_trusted_certificate = { 0, NULL }; */ + conf->buffer_size = NGX_CONF_UNSET_SIZE; + conf->max_response_body_size = NGX_CONF_UNSET_SIZE; + conf->timeout = NGX_CONF_UNSET_MSEC; + #if (NGX_STREAM_SSL) + conf->ssl_verify = NGX_CONF_UNSET; conf->ssl_verify_depth = NGX_CONF_UNSET; #endif return conf; @@ -1915,6 +1998,11 @@ ngx_stream_js_merge_srv_conf(ngx_conf_t ngx_conf_merge_str_value(conf->preread, prev->preread, ""); ngx_conf_merge_str_value(conf->filter, prev->filter, ""); + ngx_conf_merge_msec_value(conf->timeout, prev->timeout, 60000); + ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size, 16384); + ngx_conf_merge_size_value(conf->max_response_body_size, + prev->max_response_body_size, 1048576); + #if (NGX_STREAM_SSL) ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); @@ -1922,6 +2010,7 @@ ngx_stream_js_merge_srv_conf(ngx_conf_t (NGX_CONF_BITMASK_SET|NGX_SSL_TLSv1 |NGX_SSL_TLSv1_1|NGX_SSL_TLSv1_2)); + ngx_conf_merge_value(conf->ssl_verify, prev->ssl_verify, 1); ngx_conf_merge_value(conf->ssl_verify_depth, prev->ssl_verify_depth, 100); ngx_conf_merge_str_value(conf->ssl_trusted_certificate, @@ -2022,3 +2111,18 @@ ngx_stream_js_ssl(njs_vm_t *vm, ngx_stre return NULL; #endif } + + +static ngx_flag_t +ngx_stream_js_ssl_verify(njs_vm_t *vm, ngx_stream_session_t *s) +{ +#if (NGX_STREAM_SSL) + ngx_stream_js_srv_conf_t *jscf; + + jscf = ngx_stream_get_module_srv_conf(s, ngx_stream_js_module); + + return jscf->ssl_verify; +#else + return 0; +#endif +} diff -r 251e7ab400a8 -r b20de7bcee61 ts/ngx_core.d.ts --- a/ts/ngx_core.d.ts Wed Apr 27 16:31:00 2022 -0700 +++ b/ts/ngx_core.d.ts Thu Apr 28 16:37:14 2022 -0700 @@ -78,8 +78,9 @@ interface NgxFetchOptions { */ body?: NjsStringLike, /** - * The buffer size for reading the response, by default is 4096. + * The buffer size for reading the response, by default is 16384 (4096 before 0.7.4). * Nginx specific. + * @deprecated Use `js_fetch_buffer_size` directive instead. */ buffer_size?: Number, /** @@ -87,8 +88,9 @@ interface NgxFetchOptions { */ headers?: Object, /** - * The maximum size of the response body in bytes, by default is 32768. + * The maximum size of the response body in bytes, by default is 1048576 (32768 before 0.7.4). * Nginx specific. + * @deprecated Use `js_fetch_max_response_buffer_size` directive instead. */ max_response_body_size?: Number, /** From xeioex at nginx.com Fri Apr 29 00:51:35 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 29 Apr 2022 00:51:35 +0000 Subject: [njs] HTTP: expect escaped URIs in r.internalRedirect(). Message-ID: details: https://hg.nginx.org/njs/rev/280e866c2973 branches: changeset: 1845:280e866c2973 user: Dmitry Volyntsev date: Thu Apr 28 17:23:02 2022 -0700 description: HTTP: expect escaped URIs in r.internalRedirect(). Similarly to the nginx change in 975d7ab37b39 (1.17.2), we should accept properly escaped URIs and unescape them as needed, else it is not possible to handle URIs with question marks. Previously, the URI was used as is. diffstat: nginx/ngx_http_js_module.c | 15 +++++++++++++-- 1 files changed, 13 insertions(+), 2 deletions(-) diffs (32 lines): diff -r b20de7bcee61 -r 280e866c2973 nginx/ngx_http_js_module.c --- a/nginx/ngx_http_js_module.c Thu Apr 28 16:37:14 2022 -0700 +++ b/nginx/ngx_http_js_module.c Thu Apr 28 17:23:02 2022 -0700 @@ -890,7 +890,9 @@ ngx_http_js_content_write_event_handler( static void ngx_http_js_content_finalize(ngx_http_request_t *r, ngx_http_js_ctx_t *ctx) { - ngx_str_t args; + ngx_str_t args; + ngx_int_t rc; + ngx_uint_t flags; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http js content rc: %i", ctx->status); @@ -900,7 +902,16 @@ ngx_http_js_content_finalize(ngx_http_re ngx_http_named_location(r, &ctx->redirect_uri); } else { - ngx_http_split_args(r, &ctx->redirect_uri, &args); + ngx_str_null(&args); + flags = NGX_HTTP_LOG_UNSAFE; + + rc = ngx_http_parse_unsafe_uri(r, &ctx->redirect_uri, &args, + &flags); + if (rc != NGX_OK) { + ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); + return; + } + ngx_http_internal_redirect(r, &ctx->redirect_uri, &args); } } From xeioex at nginx.com Fri Apr 29 00:51:37 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 29 Apr 2022 00:51:37 +0000 Subject: [njs] Optimising JSON.parse() for large objects. Message-ID: details: https://hg.nginx.org/njs/rev/4d71bdf8663d branches: changeset: 1846:4d71bdf8663d user: Dmitry Volyntsev date: Thu Apr 28 17:49:59 2022 -0700 description: Optimising JSON.parse() for large objects. diffstat: src/njs_json.c | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diffs (28 lines): diff -r 280e866c2973 -r 4d71bdf8663d src/njs_json.c --- a/src/njs_json.c Thu Apr 28 17:23:02 2022 -0700 +++ b/src/njs_json.c Thu Apr 28 17:49:59 2022 -0700 @@ -877,8 +877,14 @@ njs_json_push_parse_state(njs_vm_t *vm, njs_inline njs_json_state_t * -njs_json_pop_parse_state(njs_json_parse_t *parse) +njs_json_pop_parse_state(njs_vm_t *vm, njs_json_parse_t *parse) { + njs_json_state_t *state; + + state = &parse->states[parse->depth - 1]; + njs_array_destroy(vm, state->keys); + state->keys = NULL; + if (parse->depth > 1) { parse->depth--; return &parse->states[parse->depth - 1]; @@ -956,7 +962,7 @@ njs_json_parse_iterator(njs_vm_t *vm, nj } } else { - state = njs_json_pop_parse_state(parse); + state = njs_json_pop_parse_state(vm, parse); if (state == NULL) { vm->retval = parse->retval; return NJS_OK; From xeioex at nginx.com Fri Apr 29 03:30:27 2022 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 29 Apr 2022 03:30:27 +0000 Subject: [njs] Added njs.version_number property. Message-ID: details: https://hg.nginx.org/njs/rev/900d3ba62bcb branches: changeset: 1847:900d3ba62bcb user: Dmitry Volyntsev date: Thu Apr 28 17:59:03 2022 -0700 description: Added njs.version_number property. diffstat: src/njs.h | 1 + src/njs_builtin.c | 8 ++++++++ src/test/njs_unit_test.c | 7 ++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diffs (53 lines): diff -r 4d71bdf8663d -r 900d3ba62bcb src/njs.h --- a/src/njs.h Thu Apr 28 17:49:59 2022 -0700 +++ b/src/njs.h Thu Apr 28 17:59:03 2022 -0700 @@ -12,6 +12,7 @@ #include #define NJS_VERSION "0.7.4" +#define NJS_VERSION_NUMBER 0x000704 #include /* STDOUT_FILENO, STDERR_FILENO */ diff -r 4d71bdf8663d -r 900d3ba62bcb src/njs_builtin.c --- a/src/njs_builtin.c Thu Apr 28 17:49:59 2022 -0700 +++ b/src/njs_builtin.c Thu Apr 28 17:59:03 2022 -0700 @@ -1693,6 +1693,14 @@ static const njs_object_prop_t njs_njs_ { .type = NJS_PROPERTY, + .name = njs_string("version_number"), + .value = njs_value(NJS_NUMBER, 1, NJS_VERSION_NUMBER), + .configurable = 1, + .enumerable = 1, + }, + + { + .type = NJS_PROPERTY, .name = njs_string("dump"), .value = njs_native_function(njs_ext_dump, 0), .configurable = 1, diff -r 4d71bdf8663d -r 900d3ba62bcb src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Thu Apr 28 17:49:59 2022 -0700 +++ b/src/test/njs_unit_test.c Thu Apr 28 17:59:03 2022 -0700 @@ -13448,6 +13448,10 @@ static njs_unit_test_t njs_test[] = { njs_str("Object.values(njs)[0] === njs.version"), njs_str("true") }, + { njs_str("njs.version.split('.')" + ".map(v => parseInt(v)).reduce((p, c) => p * 256 + c) == njs.version_number"), + njs_str("true") }, + { njs_str("Object.values(process)"), njs_str("") }, @@ -17644,9 +17648,6 @@ static njs_unit_test_t njs_test[] = { njs_str("var o = Object.defineProperty({}, 'a', { set(){}, enumerable: true }); njs.dump(o)"), njs_str("{a:'[Setter]'}") }, - { njs_str("njs.dump(njs) == `njs {version:'${njs.version}'}`"), - njs_str("true") }, - { njs_str("var a = []; a[0] = a; njs.dump(a)"), njs_str("[[Circular]]") }, From shanlei at asiainfo.com Fri Apr 29 05:25:23 2022 From: shanlei at asiainfo.com (lei shan) Date: Fri, 29 Apr 2022 13:25:23 +0800 Subject: patch: call modules' exit_process in reverse order In-Reply-To: References: <564de12e05a6d3b229e395855712255f@asiainfo.com> Message-ID: hello,Dounin, I also had read many discussions , confusing how to properly maintain the dependency orders. So despite the usage of thread pool module, I think it's a principal to call the plugin's construction , then call their destruction functions in reverse order, when we design some kind of dynamic plugin mechanism. Does that make sense? Or nginx has some consideration to force destruction follow same execution order with construction? Thanks & Regards On Thu, 28 Apr 2022, 07:55 Maxim Dounin, wrote: > Hello! > > On Sun, Apr 24, 2022 at 04:33:04PM +0800, shanlei at asiainfo.com wrote: > > > # HG changeset patch > > # User stdanley > > # Date 1650788278 -28800 > > # Sun Apr 24 16:17:58 2022 +0800 > > # Node ID 522acbe88486d027383075c8208edd6fcc0a3aa6 > > # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b > > patch: call modules' exit_process in reverse order to solve module > > dependency. > > We once have developed a module which depends on ngx_thread_pool module. > > but nginx hungs when began to shutdown. trace shows that ngx_thread_pool > > cant finish "exit_process" because our module is using a thread, our > > module must > > call "exit_process" before ngx_thread_pool. > > so we present this patch to solve modules' dependency issue. > > Thank you for the patch. > > First of all, you may want to consider this section in the > development guide: > > http://nginx.org/en/docs/dev/development_guide.html#threads_pitfalls > > Using threads in nginx is usually a bad idea. Thread pools, in > particular, are designed to be used to offload short operations like > read() syscalls. And these operations are expected to be finished > before nginx exit, so thread pool's exit handler waits for all > threads to complete. > > If you are sure you have to use threads, and thread pools in > particular, and with some long-running tasks, you may want to > consider other ways to signal your threads to exit. In > particular, checking ngx_exiting/ngx_quit/ngx_terminate > periodically might be a way to go. Though I would recommend to > reconsider using long-running thread tasks anyway. > > As for the patch, it looks wrong to me. While changing the > exit_process() callback order might solve your particular > dependency issue, similar issues might easily appear with any > order. > > Hope this helps. > > -- > Maxim Dounin > http://mdounin.ru/ > _______________________________________________ > nginx-devel mailing list -- nginx-devel at nginx.org > To unsubscribe send an email to nginx-devel-leave at nginx.org > -------------- next part -------------- An HTML attachment was scrubbed... URL: From pluknet at nginx.com Fri Apr 29 13:39:18 2022 From: pluknet at nginx.com (=?iso-8859-1?q?Sergey_Kandaurov?=) Date: Fri, 29 Apr 2022 17:39:18 +0400 Subject: [PATCH] Configure: recognize arm64 machine name as a synonym for aarch64 Message-ID: <35afae4b3dffff6718c0.1651239558@enoparse.local> # HG changeset patch # User Sergey Kandaurov # Date 1651239481 -14400 # Fri Apr 29 17:38:01 2022 +0400 # Node ID 35afae4b3dffff6718c0cab3ceb16b9de207c20a # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b Configure: recognize arm64 machine name as a synonym for aarch64. In particular, this sets a reasonable cacheline size on FreeBSD and macOS, which prefer to use this name and both lack _SC_LEVEL1_DCACHE_LINESIZE. diff --git a/auto/os/conf b/auto/os/conf --- a/auto/os/conf +++ b/auto/os/conf @@ -110,7 +110,7 @@ case "$NGX_MACHINE" in NGX_MACH_CACHE_LINE=64 ;; - aarch64 ) + aarch64 | arm64) have=NGX_ALIGNMENT value=16 . auto/define NGX_MACH_CACHE_LINE=64 ;; From rekgrpth at gmail.com Sat Apr 30 05:03:42 2022 From: rekgrpth at gmail.com (=?iso-8859-1?q?RekGRpth?=) Date: Sat, 30 Apr 2022 10:03:42 +0500 Subject: [PATCH] upstream keepalive: save and restore connection data Message-ID: <7b38618adf6317fc2683.1651295022@gws.t72.ru> # HG changeset patch # User RekGRpth # Date 1651294527 -18000 # Sat Apr 30 09:55:27 2022 +0500 # Node ID 7b38618adf6317fc268315b8134a5e2b1bc96269 # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b upstream keepalive: save and restore connection data save connection data before keep it and restore connection data after yuild it diff -r a736a7a613ea -r 7b38618adf63 src/http/modules/ngx_http_upstream_keepalive_module.c --- a/src/http/modules/ngx_http_upstream_keepalive_module.c Tue Feb 08 17:35:27 2022 +0300 +++ b/src/http/modules/ngx_http_upstream_keepalive_module.c Sat Apr 30 09:55:27 2022 +0500 @@ -34,6 +34,8 @@ socklen_t socklen; ngx_sockaddr_t sockaddr; + void *data; + } ngx_http_upstream_keepalive_cache_t; @@ -284,7 +286,7 @@ c->idle = 0; c->sent = 0; - c->data = NULL; + c->data = item->data; c->log = pc->log; c->read->log = pc->log; c->write->log = pc->log; @@ -390,6 +392,7 @@ c->write->handler = ngx_http_upstream_keepalive_dummy_handler; c->read->handler = ngx_http_upstream_keepalive_close_handler; + item->data = c->data; c->data = item; c->idle = 1; c->log = ngx_cycle->log; From mdounin at mdounin.ru Sat Apr 30 22:19:01 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sun, 1 May 2022 01:19:01 +0300 Subject: [PATCH] upstream keepalive: save and restore connection data In-Reply-To: <7b38618adf6317fc2683.1651295022@gws.t72.ru> References: <7b38618adf6317fc2683.1651295022@gws.t72.ru> Message-ID: Hello! On Sat, Apr 30, 2022 at 10:03:42AM +0500, RekGRpth wrote: > # HG changeset patch > # User RekGRpth > # Date 1651294527 -18000 > # Sat Apr 30 09:55:27 2022 +0500 > # Node ID 7b38618adf6317fc268315b8134a5e2b1bc96269 > # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b > upstream keepalive: save and restore connection data > > save connection data before keep it > and > restore connection data after yuild it > > diff -r a736a7a613ea -r 7b38618adf63 src/http/modules/ngx_http_upstream_keepalive_module.c > --- a/src/http/modules/ngx_http_upstream_keepalive_module.c Tue Feb 08 17:35:27 2022 +0300 > +++ b/src/http/modules/ngx_http_upstream_keepalive_module.c Sat Apr 30 09:55:27 2022 +0500 > @@ -34,6 +34,8 @@ > socklen_t socklen; > ngx_sockaddr_t sockaddr; > > + void *data; > + > } ngx_http_upstream_keepalive_cache_t; > > > @@ -284,7 +286,7 @@ > > c->idle = 0; > c->sent = 0; > - c->data = NULL; > + c->data = item->data; > c->log = pc->log; > c->read->log = pc->log; > c->write->log = pc->log; > @@ -390,6 +392,7 @@ > c->write->handler = ngx_http_upstream_keepalive_dummy_handler; > c->read->handler = ngx_http_upstream_keepalive_close_handler; > > + item->data = c->data; > c->data = item; > c->idle = 1; > c->log = ngx_cycle->log; Thanks for the patch. You may want to be more specific on what problem you are trying to solve by this patch. -- Maxim Dounin http://mdounin.ru/ From mdounin at mdounin.ru Sat Apr 30 22:24:20 2022 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sun, 1 May 2022 01:24:20 +0300 Subject: [PATCH] Configure: recognize arm64 machine name as a synonym for aarch64 In-Reply-To: <35afae4b3dffff6718c0.1651239558@enoparse.local> References: <35afae4b3dffff6718c0.1651239558@enoparse.local> Message-ID: Hello! On Fri, Apr 29, 2022 at 05:39:18PM +0400, Sergey Kandaurov wrote: > # HG changeset patch > # User Sergey Kandaurov > # Date 1651239481 -14400 > # Fri Apr 29 17:38:01 2022 +0400 > # Node ID 35afae4b3dffff6718c0cab3ceb16b9de207c20a > # Parent a736a7a613ea6e182ff86fbadcb98bb0f8891c0b > Configure: recognize arm64 machine name as a synonym for aarch64. > > In particular, this sets a reasonable cacheline size on FreeBSD and macOS, > which prefer to use this name and both lack _SC_LEVEL1_DCACHE_LINESIZE. > > diff --git a/auto/os/conf b/auto/os/conf > --- a/auto/os/conf > +++ b/auto/os/conf > @@ -110,7 +110,7 @@ case "$NGX_MACHINE" in > NGX_MACH_CACHE_LINE=64 > ;; > > - aarch64 ) > + aarch64 | arm64) > have=NGX_ALIGNMENT value=16 . auto/define > NGX_MACH_CACHE_LINE=64 > ;; Looks good. -- Maxim Dounin http://mdounin.ru/