From xeioex at nginx.com  Thu Jul  2 14:01:35 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 02 Jul 2020 14:01:35 +0000
Subject: [njs] Parser: fixed parsing of regexp flags.
Message-ID: <hg.6345c07f1c52.1593698495.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/6345c07f1c52
branches:  
changeset: 1444:6345c07f1c52
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu Jul 02 12:58:50 2020 +0000
description:
Parser: fixed parsing of regexp flags.

diffstat:

 src/njs_parser.c         |   2 +-
 src/njs_regexp.c         |  32 ++++++++++----------------------
 src/njs_regexp.h         |   4 ++--
 src/test/njs_unit_test.c |  21 ++++++++++++++++++---
 4 files changed, 31 insertions(+), 28 deletions(-)

diffs (143 lines):

diff -r e8a941b394a3 -r 6345c07f1c52 src/njs_parser.c
--- a/src/njs_parser.c	Tue Jun 30 15:36:31 2020 +0000
+++ b/src/njs_parser.c	Thu Jul 02 12:58:50 2020 +0000
@@ -1247,7 +1247,7 @@ njs_parser_regexp_literal(njs_parser_t *
             p++;
             lexer->start = p;
 
-            flags = njs_regexp_flags(&p, lexer->end, 0);
+            flags = njs_regexp_flags(&p, lexer->end);
 
             if (njs_slow_path(flags < 0)) {
                 njs_parser_syntax_error(parser, "Invalid RegExp flags \"%*s\"",
diff -r e8a941b394a3 -r 6345c07f1c52 src/njs_regexp.c
--- a/src/njs_regexp.c	Tue Jun 30 15:36:31 2020 +0000
+++ b/src/njs_regexp.c	Thu Jul 02 12:58:50 2020 +0000
@@ -143,8 +143,10 @@ njs_regexp_constructor(njs_vm_t *vm, njs
 
         start = string.start;
 
-        re_flags = njs_regexp_flags(&start, start + string.length, 1);
-        if (njs_slow_path(re_flags < 0)) {
+        re_flags = njs_regexp_flags(&start, start + string.length);
+        if (njs_slow_path(re_flags < 0
+                          || (size_t) (start - string.start) != string.length))
+        {
             njs_syntax_error(vm, "Invalid RegExp flags \"%V\"", &string);
             return NJS_ERROR;
         }
@@ -305,17 +307,15 @@ done:
 
 
 njs_regexp_flags_t
-njs_regexp_flags(u_char **start, u_char *end, njs_bool_t bound)
+njs_regexp_flags(u_char **start, u_char *end)
 {
     u_char              *p;
     njs_regexp_flags_t  flags, flag;
 
-    flags = 0;
+    flags = NJS_REGEXP_NO_FLAGS;
 
     for (p = *start; p < end; p++) {
-
         switch (*p) {
-
         case 'g':
             flag = NJS_REGEXP_GLOBAL;
             break;
@@ -328,24 +328,12 @@ njs_regexp_flags(u_char **start, u_char 
             flag = NJS_REGEXP_MULTILINE;
             break;
 
-        case ';':
-        case ' ':
-        case '\t':
-        case '\r':
-        case '\n':
-        case ',':
-        case ')':
-        case ']':
-        case '}':
-        case '.':
-            if (!bound) {
-                goto done;
+        default:
+            if (*p >= 'a' && *p <= 'z') {
+                goto invalid;
             }
 
-            /* Fall through. */
-
-        default:
-            goto invalid;
+            goto done;
         }
 
         if (njs_slow_path((flags & flag) != 0)) {
diff -r e8a941b394a3 -r 6345c07f1c52 src/njs_regexp.h
--- a/src/njs_regexp.h	Tue Jun 30 15:36:31 2020 +0000
+++ b/src/njs_regexp.h	Thu Jul 02 12:58:50 2020 +0000
@@ -10,6 +10,7 @@
 
 typedef enum {
     NJS_REGEXP_INVALID_FLAG = -1,
+    NJS_REGEXP_NO_FLAGS     =  0,
     NJS_REGEXP_GLOBAL       =  1,
     NJS_REGEXP_IGNORE_CASE  =  2,
     NJS_REGEXP_MULTILINE    =  4,
@@ -19,8 +20,7 @@ typedef enum {
 njs_int_t njs_regexp_init(njs_vm_t *vm);
 njs_int_t njs_regexp_create(njs_vm_t *vm, njs_value_t *value, u_char *start,
     size_t length, njs_regexp_flags_t flags);
-njs_regexp_flags_t njs_regexp_flags(u_char **start, u_char *end,
-    njs_bool_t bound);
+njs_regexp_flags_t njs_regexp_flags(u_char **start, u_char *end);
 njs_regexp_pattern_t *njs_regexp_pattern_create(njs_vm_t *vm,
     u_char *string, size_t length, njs_regexp_flags_t flags);
 njs_int_t njs_regexp_match(njs_vm_t *vm, njs_regex_t *regex,
diff -r e8a941b394a3 -r 6345c07f1c52 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Tue Jun 30 15:36:31 2020 +0000
+++ b/src/test/njs_unit_test.c	Thu Jul 02 12:58:50 2020 +0000
@@ -9487,9 +9487,6 @@ static njs_unit_test_t  njs_test[] =
 
     /* RegExp. */
 
-    { njs_str("/./x"),
-      njs_str("SyntaxError: Invalid RegExp flags \"x\" in 1") },
-
     { njs_str("/"),
       njs_str("SyntaxError: Unterminated RegExp \"/\" in 1") },
 
@@ -9526,6 +9523,15 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("/\\\n/"),
       njs_str("SyntaxError: Unterminated RegExp \"/\\\" in 1") },
 
+    { njs_str("/./x"),
+      njs_str("SyntaxError: Invalid RegExp flags \"x\" in 1") },
+
+    { njs_str("/./.exec === RegExp.prototype.exec"),
+      njs_str("true") },
+
+    { njs_str("/./['exec'] === RegExp.prototype.exec"),
+      njs_str("true") },
+
     { njs_str("/^[A-Za-z0-9+/]{4}$/.test('////')"),
       njs_str("true") },
 
@@ -9736,6 +9742,15 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("new RegExp('', 'x')"),
       njs_str("SyntaxError: Invalid RegExp flags \"x\"") },
 
+    { njs_str("new RegExp('', 'g ')"),
+      njs_str("SyntaxError: Invalid RegExp flags \"g \"") },
+
+    { njs_str("new RegExp('', '')"),
+      njs_str("/(?:)/") },
+
+    { njs_str("new RegExp('', {toString:()=>'g'})"),
+      njs_str("/(?:)/g") },
+
     { njs_str("RegExp({})"),
       njs_str("/[object Object]/") },
 

From xeioex at nginx.com  Thu Jul  2 14:01:37 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 02 Jul 2020 14:01:37 +0000
Subject: [njs] Fixed String.prototype.repeat() according to the specification.
Message-ID: <hg.27b88bd86e61.1593698497.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/27b88bd86e61
branches:  
changeset: 1445:27b88bd86e61
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu Jul 02 12:59:48 2020 +0000
description:
Fixed String.prototype.repeat() according to the specification.

diffstat:

 src/njs_string.c         |  42 ++++++++++++++++++++++++++++--------------
 src/test/njs_unit_test.c |   8 +++++---
 2 files changed, 33 insertions(+), 17 deletions(-)

diffs (107 lines):

diff -r 6345c07f1c52 -r 27b88bd86e61 src/njs_string.c
--- a/src/njs_string.c	Thu Jul 02 12:58:50 2020 +0000
+++ b/src/njs_string.c	Thu Jul 02 12:59:48 2020 +0000
@@ -2772,47 +2772,61 @@ static njs_int_t
 njs_string_prototype_repeat(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     njs_index_t unused)
 {
-    u_char             *p, *start;
+    u_char             *p;
+    double             count;
     int64_t            n, max;
     uint64_t           size, length;
     njs_int_t          ret;
+    njs_value_t        *this;
     njs_string_prop_t  string;
 
-    ret = njs_string_object_validate(vm, njs_arg(args, nargs, 0));
+    this = njs_argument(args, 0);
+
+    if (njs_slow_path(njs_is_null_or_undefined(this))) {
+        njs_type_error(vm, "cannot convert \"%s\"to object",
+                       njs_type_string(this->type));
+        return NJS_ERROR;
+    }
+
+    ret = njs_value_to_string(vm, this, this);
     if (njs_slow_path(ret != NJS_OK)) {
         return ret;
     }
 
-    ret = njs_value_to_integer(vm, njs_arg(args, nargs, 1), &n);
+    ret = njs_value_to_number(vm, njs_arg(args, nargs, 1), &count);
     if (njs_slow_path(ret != NJS_OK)) {
         return ret;
     }
 
-    (void) njs_string_prop(&string, njs_argument(args, 0));
-
-    max = (string.size > 1) ? NJS_STRING_MAX_LENGTH / string.size
-                            : NJS_STRING_MAX_LENGTH;
-
-    if (njs_slow_path(n < 0 || n >= max)) {
+    if (njs_slow_path(!isnan(count) && (count < 0 || isinf(count)))) {
         njs_range_error(vm, NULL);
         return NJS_ERROR;
     }
 
-    if (string.size == 0) {
+     n = njs_number_to_integer(count);
+
+    (void) njs_string_prop(&string, this);
+
+    if (njs_slow_path(n == 0 || string.size == 0)) {
         vm->retval = njs_string_empty;
         return NJS_OK;
     }
 
+    max = NJS_STRING_MAX_LENGTH / string.size;
+
+    if (njs_slow_path(n >= max)) {
+        njs_range_error(vm, NULL);
+        return NJS_ERROR;
+    }
+
     size = string.size * n;
     length = string.length * n;
 
-    start = njs_string_alloc(vm, &vm->retval, size, length);
-    if (njs_slow_path(start == NULL)) {
+    p = njs_string_alloc(vm, &vm->retval, size, length);
+    if (njs_slow_path(p == NULL)) {
         return NJS_ERROR;
     }
 
-    p = start;
-
     while (n != 0) {
         p = memcpy(p, string.start, string.size);
         p += string.size;
diff -r 6345c07f1c52 -r 27b88bd86e61 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Thu Jul 02 12:58:50 2020 +0000
+++ b/src/test/njs_unit_test.c	Thu Jul 02 12:59:48 2020 +0000
@@ -7892,12 +7892,11 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("''.repeat(2147483646)"),
       njs_str("") },
 
-    /* ES6: "". */
     { njs_str("''.repeat(2147483647)"),
-      njs_str("RangeError") },
+      njs_str("") },
 
     { njs_str("''.repeat(2147483648)"),
-      njs_str("RangeError") },
+      njs_str("") },
 
     { njs_str("''.repeat(Infinity)"),
       njs_str("RangeError") },
@@ -7905,6 +7904,9 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("''.repeat(NaN)"),
       njs_str("") },
 
+    { njs_str("String.prototype.repeat.call({},2)"),
+      njs_str("[object Object][object Object]") },
+
     { njs_str("'abc'.padStart(7)"),
       njs_str("    abc") },
 

From xeioex at nginx.com  Thu Jul  2 14:01:39 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 02 Jul 2020 14:01:39 +0000
Subject: [njs] Improved README page.
Message-ID: <hg.c30a2a805014.1593698499.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/c30a2a805014
branches:  
changeset: 1446:c30a2a805014
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu Jul 02 12:59:54 2020 +0000
description:
Improved README page.

diffstat:

 README |  9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diffs (20 lines):

diff -r 27b88bd86e61 -r c30a2a805014 README
--- a/README	Thu Jul 02 12:59:48 2020 +0000
+++ b/README	Thu Jul 02 12:59:54 2020 +0000
@@ -1,3 +1,11 @@
+
+NGINX JavaScript (njs)
+----------
+
+njs is a subset of the JavaScript language that allows extending nginx
+functionality. njs is created in compliance with ECMAScript 5.1 (strict mode)
+with some ECMAScript 6 and later extensions. The compliance is still evolving.
+
 The documentation is available online:
 
   http://nginx.org/en/docs/njs/
@@ -12,3 +20,4 @@ or via Github:
 
 --
 NGINX, Inc., http://nginx.com
+

From xeioex at nginx.com  Thu Jul  2 14:01:41 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 02 Jul 2020 14:01:41 +0000
Subject: [njs] Introduced RegExpBuiltinExec().
Message-ID: <hg.f9082cd59ba6.1593698501.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/f9082cd59ba6
branches:  
changeset: 1447:f9082cd59ba6
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu Jul 02 13:59:33 2020 +0000
description:
Introduced RegExpBuiltinExec().

diffstat:

 src/njs_pcre.c           |    6 +-
 src/njs_regex.h          |    3 +-
 src/njs_regexp.c         |  277 +++++++++++++++++++++++++++-------------------
 src/njs_regexp.h         |    4 +-
 src/njs_string.c         |    8 +-
 src/test/njs_unit_test.c |    3 +
 6 files changed, 180 insertions(+), 121 deletions(-)

diffs (523 lines):

diff -r c30a2a805014 -r f9082cd59ba6 src/njs_pcre.c
--- a/src/njs_pcre.c	Thu Jul 02 12:59:54 2020 +0000
+++ b/src/njs_pcre.c	Thu Jul 02 13:59:33 2020 +0000
@@ -280,13 +280,13 @@ njs_pcre_default_free(void *p, void *mem
 
 
 njs_int_t
-njs_regex_match(njs_regex_t *regex, const u_char *subject, size_t len,
-    njs_regex_match_data_t *match_data, njs_regex_context_t *ctx)
+njs_regex_match(njs_regex_t *regex, const u_char *subject, size_t off,
+    size_t len, njs_regex_match_data_t *match_data, njs_regex_context_t *ctx)
 {
     int  ret;
 
     ret = pcre_exec(regex->code, regex->extra, (const char *) subject, len,
-                    0, 0, match_data->captures, match_data->ncaptures);
+                    off, 0, match_data->captures, match_data->ncaptures);
 
     /* PCRE_ERROR_NOMATCH is -1. */
 
diff -r c30a2a805014 -r f9082cd59ba6 src/njs_regex.h
--- a/src/njs_regex.h	Thu Jul 02 12:59:54 2020 +0000
+++ b/src/njs_regex.h	Thu Jul 02 13:59:33 2020 +0000
@@ -39,7 +39,8 @@ NJS_EXPORT njs_regex_match_data_t *njs_r
 NJS_EXPORT void njs_regex_match_data_free(njs_regex_match_data_t *match_data,
     njs_regex_context_t *ctx);
 NJS_EXPORT njs_int_t njs_regex_match(njs_regex_t *regex, const u_char *subject,
-    size_t len, njs_regex_match_data_t *match_data, njs_regex_context_t *ctx);
+    size_t off, size_t len, njs_regex_match_data_t *match_data,
+    njs_regex_context_t *ctx);
 NJS_EXPORT int *njs_regex_captures(njs_regex_match_data_t *match_data);
 
 
diff -r c30a2a805014 -r f9082cd59ba6 src/njs_regexp.c
--- a/src/njs_regexp.c	Thu Jul 02 12:59:54 2020 +0000
+++ b/src/njs_regexp.c	Thu Jul 02 13:59:33 2020 +0000
@@ -26,9 +26,9 @@ static u_char *njs_regexp_compile_trace_
     njs_trace_data_t *td, u_char *start);
 static u_char *njs_regexp_match_trace_handler(njs_trace_t *trace,
     njs_trace_data_t *td, u_char *start);
-static njs_int_t njs_regexp_exec_result(njs_vm_t *vm, njs_regexp_t *regexp,
-    njs_utf8_t utf8, u_char *string, njs_regex_match_data_t *match_data,
-    uint32_t last_index);
+static njs_array_t *njs_regexp_exec_result(njs_vm_t *vm, njs_regexp_t *regexp,
+    njs_regexp_utf8_t type, njs_string_prop_t *string,
+    njs_regex_match_data_t *data);
 static njs_int_t njs_regexp_string_create(njs_vm_t *vm, njs_value_t *value,
     u_char *start, uint32_t size, int32_t length);
 
@@ -550,7 +550,7 @@ njs_regexp_compile_trace_handler(njs_tra
 
 njs_int_t
 njs_regexp_match(njs_vm_t *vm, njs_regex_t *regex, const u_char *subject,
-    size_t len, njs_regex_match_data_t *match_data)
+    size_t off, size_t len, njs_regex_match_data_t *match_data)
 {
     njs_int_t            ret;
     njs_trace_handler_t  handler;
@@ -558,7 +558,8 @@ njs_regexp_match(njs_vm_t *vm, njs_regex
     handler = vm->trace.handler;
     vm->trace.handler = njs_regexp_match_trace_handler;
 
-    ret = njs_regex_match(regex, subject, len, match_data, vm->regex_context);
+    ret = njs_regex_match(regex, subject, off, len, match_data,
+                          vm->regex_context);
 
     vm->trace.handler = handler;
 
@@ -617,9 +618,7 @@ static njs_int_t
 njs_regexp_prototype_last_index(njs_vm_t *vm, njs_object_prop_t *unused,
     njs_value_t *value, njs_value_t *setval, njs_value_t *retval)
 {
-    uint32_t           index, last_index;
-    njs_regexp_t       *regexp;
-    njs_string_prop_t  string;
+    njs_regexp_t  *regexp;
 
     regexp = njs_object_proto_lookup(njs_object(value), NJS_REGEXP,
                                      njs_regexp_t);
@@ -635,23 +634,7 @@ njs_regexp_prototype_last_index(njs_vm_t
         return NJS_OK;
     }
 
-    if (njs_slow_path(!njs_is_number(&regexp->last_index))) {
-        *retval = regexp->last_index;
-        return NJS_OK;
-    }
-
-    (void) njs_string_prop(&string, &regexp->string);
-
-    last_index = njs_number(&regexp->last_index);
-
-    if (njs_slow_path(string.size < last_index)) {
-        *retval = regexp->last_index;
-        return NJS_OK;
-    }
-
-    index = njs_string_index(&string, last_index);
-    njs_set_number(retval, index);
-
+    *retval = regexp->last_index;
     return NJS_OK;
 }
 
@@ -802,8 +785,8 @@ njs_regexp_prototype_test(njs_vm_t *vm, 
             }
         }
 
-        match = njs_regexp_match(vm, regex, string.start, string.size,
-                               match_data);
+        match = njs_regexp_match(vm, regex, string.start, 0, string.size,
+                                 match_data);
         if (match >= 0) {
             retval = &njs_value_true;
 
@@ -844,36 +827,25 @@ done:
 }
 
 
-njs_int_t
-njs_regexp_prototype_exec(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
-    njs_index_t unused)
+/**
+ * TODO: sticky, unicode flags.
+ */
+static njs_int_t
+njs_regexp_builtin_exec(njs_vm_t *vm, njs_value_t *r, njs_value_t *s,
+    njs_value_t *retval)
 {
+    size_t                  length, offset;
     int64_t                 last_index;
     njs_int_t               ret;
-    njs_utf8_t              utf8;
-    njs_value_t             *value, lvalue;
+    njs_array_t             *result;
     njs_regexp_t            *regexp;
     njs_string_prop_t       string;
     njs_regexp_utf8_t       type;
     njs_regexp_pattern_t    *pattern;
     njs_regex_match_data_t  *match_data;
 
-    if (!njs_is_regexp(njs_arg(args, nargs, 0))) {
-        njs_type_error(vm, "\"this\" argument is not a regexp");
-        return NJS_ERROR;
-    }
-
-    value = njs_lvalue_arg(&lvalue, args, nargs, 1);
-
-    if (!njs_is_string(value)) {
-        ret = njs_value_to_string(vm, value, value);
-        if (njs_slow_path(ret != NJS_OK)) {
-            return ret;
-        }
-    }
-
-    regexp = njs_regexp(&args[0]);
-    regexp->string = *value;
+    regexp = njs_regexp(r);
+    regexp->string = *s;
     pattern = regexp->pattern;
 
     ret = njs_value_to_length(vm, &regexp->last_index, &last_index);
@@ -885,94 +857,113 @@ njs_regexp_prototype_exec(njs_vm_t *vm, 
         last_index = 0;
     }
 
-    (void) njs_string_prop(&string, value);
+    length = njs_string_prop(&string, s);
+
+    if (njs_slow_path((size_t) last_index > length)) {
+        goto not_found;
+    }
+
+    type = NJS_REGEXP_BYTE;
 
-    if (string.size >= (size_t) last_index) {
-        utf8 = NJS_STRING_BYTE;
-        type = NJS_REGEXP_BYTE;
+    if (length != string.size) {
+        /* UTF-8 string. */
+        type = NJS_REGEXP_UTF8;
+    }
+
+    pattern = regexp->pattern;
+
+    if (njs_slow_path(!njs_regex_is_valid(&pattern->regex[type]))) {
+        goto not_found;
+    }
 
-        if (string.length != 0) {
-            utf8 = NJS_STRING_ASCII;
-            type = NJS_REGEXP_UTF8;
+    match_data = njs_regex_match_data(&pattern->regex[type], vm->regex_context);
+    if (njs_slow_path(match_data == NULL)) {
+        njs_memory_error(vm);
+        return NJS_ERROR;
+    }
+
+    if (type != NJS_REGEXP_UTF8) {
+        offset = last_index;
 
-            if (string.length != string.size) {
-                utf8 = NJS_STRING_UTF8;
-            }
+    } else {
+        /* UTF-8 string. */
+        offset = njs_string_offset(string.start, string.start + string.size,
+                                   last_index) - string.start;
+    }
+
+    ret = njs_regexp_match(vm, &pattern->regex[type], string.start, offset,
+                           string.size, match_data);
+    if (ret >= 0) {
+        result = njs_regexp_exec_result(vm, regexp, type, &string, match_data);
+        if (njs_slow_path(result == NULL)) {
+            return NJS_ERROR;
         }
 
-        pattern = regexp->pattern;
-
-        if (njs_regex_is_valid(&pattern->regex[type])) {
-            string.start += last_index;
-            string.size -= last_index;
-
-            match_data = njs_regex_match_data(&pattern->regex[type],
-                                              vm->regex_context);
-            if (njs_slow_path(match_data == NULL)) {
-                njs_memory_error(vm);
-                return NJS_ERROR;
-            }
+        njs_set_array(retval, result);
+        return NJS_OK;
+    }
 
-            ret = njs_regexp_match(vm, &pattern->regex[type], string.start,
-                                   string.size, match_data);
-            if (ret >= 0) {
-                return njs_regexp_exec_result(vm, regexp, utf8, string.start,
-                                              match_data, last_index);
-            }
+    if (njs_slow_path(ret != NJS_REGEX_NOMATCH)) {
+        njs_regex_match_data_free(match_data, vm->regex_context);
 
-            if (njs_slow_path(ret != NJS_REGEX_NOMATCH)) {
-                njs_regex_match_data_free(match_data, vm->regex_context);
+        return NJS_ERROR;
+    }
 
-                return NJS_ERROR;
-            }
-        }
-    }
+not_found:
 
     if (pattern->global) {
         njs_set_number(&regexp->last_index, 0);
     }
 
-    vm->retval = njs_value_null;
+    njs_set_null(retval);
 
     return NJS_OK;
 }
 
 
-static njs_int_t
-njs_regexp_exec_result(njs_vm_t *vm, njs_regexp_t *regexp, njs_utf8_t utf8,
-    u_char *string, njs_regex_match_data_t *match_data, uint32_t last_index)
+static njs_array_t *
+njs_regexp_exec_result(njs_vm_t *vm, njs_regexp_t *regexp,
+    njs_regexp_utf8_t type, njs_string_prop_t *string,
+    njs_regex_match_data_t *match_data)
 {
-    int                 *captures;
-    u_char              *start;
-    int32_t             size, length;
-    njs_int_t           ret;
-    njs_uint_t          i, n;
-    njs_array_t         *array;
-    njs_value_t         name;
-    njs_object_t        *groups;
-    njs_object_prop_t   *prop;
-    njs_regexp_group_t  *group;
-    njs_lvlhsh_query_t  lhq;
+    int                   *captures;
+    u_char                *start;
+    int32_t               size, length;
+    njs_int_t             ret;
+    njs_uint_t            i, n;
+    njs_array_t           *array;
+    njs_value_t           name;
+    njs_object_t          *groups;
+    njs_object_prop_t     *prop;
+    njs_regexp_group_t    *group;
+    njs_lvlhsh_query_t    lhq;
+    njs_regexp_pattern_t  *pattern;
 
     static const njs_value_t  string_index = njs_string("index");
     static const njs_value_t  string_input = njs_string("input");
     static const njs_value_t  string_groups = njs_string("groups");
 
-    array = njs_array_alloc(vm, 0, regexp->pattern->ncaptures, 0);
+    pattern = regexp->pattern;
+    array = njs_array_alloc(vm, 0, pattern->ncaptures, 0);
     if (njs_slow_path(array == NULL)) {
         goto fail;
     }
 
     captures = njs_regex_captures(match_data);
 
-    for (i = 0; i < regexp->pattern->ncaptures; i++) {
+    for (i = 0; i < pattern->ncaptures; i++) {
         n = 2 * i;
 
         if (captures[n] != -1) {
-            start = &string[captures[n]];
+            start = &string->start[captures[n]];
             size = captures[n + 1] - captures[n];
 
-            length = njs_string_calc_length(utf8, start, size);
+            if (type == NJS_REGEXP_UTF8) {
+                length = njs_max(njs_utf8_length(start, size), 0);
+
+            } else {
+                length = size;
+            }
 
             ret = njs_regexp_string_create(vm, &array->start[i], start, size,
                                            length);
@@ -985,17 +976,17 @@ njs_regexp_exec_result(njs_vm_t *vm, njs
         }
     }
 
+    /* FIXME: implement fast CreateDataPropertyOrThrow(). */
     prop = njs_object_prop_alloc(vm, &string_index, &njs_value_undefined, 1);
     if (njs_slow_path(prop == NULL)) {
         goto fail;
     }
 
-    /* TODO: Non UTF-8 position */
+    njs_set_number(&prop->value, njs_string_index(string, captures[0]));
 
-    njs_set_number(&prop->value, last_index + captures[0]);
-
-    if (regexp->pattern->global) {
-        njs_set_number(&regexp->last_index, last_index + captures[1]);
+    if (pattern->global) {
+        njs_set_number(&regexp->last_index,
+                       njs_string_index(string, captures[1]));
     }
 
     lhq.key_hash = NJS_INDEX_HASH;
@@ -1038,7 +1029,7 @@ njs_regexp_exec_result(njs_vm_t *vm, njs
         goto insert_fail;
     }
 
-    if (regexp->pattern->ngroups != 0) {
+    if (pattern->ngroups != 0) {
         groups = njs_object_alloc(vm);
         if (njs_slow_path(groups == NULL)) {
             goto fail;
@@ -1049,7 +1040,7 @@ njs_regexp_exec_result(njs_vm_t *vm, njs
         i = 0;
 
         do {
-            group = &regexp->pattern->groups[i];
+            group = &pattern->groups[i];
 
             ret = njs_string_set(vm, &name, group->name.start,
                                  group->name.length);
@@ -1074,11 +1065,9 @@ njs_regexp_exec_result(njs_vm_t *vm, njs
 
             i++;
 
-        } while (i < regexp->pattern->ngroups);
+        } while (i < pattern->ngroups);
     }
 
-    njs_set_array(&vm->retval, array);
-
     ret = NJS_OK;
     goto done;
 
@@ -1094,7 +1083,71 @@ done:
 
     njs_regex_match_data_free(match_data, vm->regex_context);
 
-    return ret;
+    return (ret == NJS_OK) ? array : NULL;
+}
+
+
+njs_int_t
+njs_regexp_prototype_exec(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
+{
+    njs_int_t    ret;
+    njs_value_t  *r, *s;
+    njs_value_t  string_lvalue;
+
+    r = njs_argument(args, 0);
+
+    if (njs_slow_path(!njs_is_regexp(r))) {
+        njs_type_error(vm, "\"this\" argument is not a regexp");
+        return NJS_ERROR;
+    }
+
+    s = njs_lvalue_arg(&string_lvalue, args, nargs, 1);
+
+    ret = njs_value_to_string(vm, s, s);
+    if (njs_slow_path(ret != NJS_OK)) {
+        return ret;
+    }
+
+    return njs_regexp_builtin_exec(vm, r, s, &vm->retval);
+}
+
+
+njs_int_t
+njs_regexp_exec(njs_vm_t *vm, njs_value_t *r, njs_value_t *s,
+    njs_value_t *retval)
+{
+    njs_int_t    ret;
+    njs_value_t  exec;
+
+    static const njs_value_t  string_exec = njs_string("exec");
+
+    ret = njs_value_property(vm, r, njs_value_arg(&string_exec), &exec);
+    if (njs_slow_path(ret == NJS_ERROR)) {
+        return NJS_ERROR;
+    }
+
+    if (njs_is_function(&exec)) {
+        ret = njs_function_call(vm, njs_function(&exec), r, s, 1, retval);
+        if (njs_slow_path(ret == NJS_ERROR)) {
+            return NJS_ERROR;
+        }
+
+        if (njs_slow_path(!njs_is_object(retval) && !njs_is_null(retval))) {
+            njs_type_error(vm, "unexpected \"%s\" retval in njs_regexp_exec()",
+                           njs_type_string(retval->type));
+            return NJS_ERROR;
+        }
+
+        return NJS_OK;
+    }
+
+    if (njs_slow_path(!njs_is_regexp(r))) {
+        njs_type_error(vm, "receiver argument is not a regexp");
+        return NJS_ERROR;
+    }
+
+    return njs_regexp_builtin_exec(vm, r, s, retval);
 }
 
 
diff -r c30a2a805014 -r f9082cd59ba6 src/njs_regexp.h
--- a/src/njs_regexp.h	Thu Jul 02 12:59:54 2020 +0000
+++ b/src/njs_regexp.h	Thu Jul 02 13:59:33 2020 +0000
@@ -24,8 +24,10 @@ njs_regexp_flags_t njs_regexp_flags(u_ch
 njs_regexp_pattern_t *njs_regexp_pattern_create(njs_vm_t *vm,
     u_char *string, size_t length, njs_regexp_flags_t flags);
 njs_int_t njs_regexp_match(njs_vm_t *vm, njs_regex_t *regex,
-    const u_char *subject, size_t len, njs_regex_match_data_t *match_data);
+    const u_char *subject, size_t off, size_t len, njs_regex_match_data_t *d);
 njs_regexp_t *njs_regexp_alloc(njs_vm_t *vm, njs_regexp_pattern_t *pattern);
+njs_int_t njs_regexp_exec(njs_vm_t *vm, njs_value_t *r, njs_value_t *s,
+    njs_value_t *retval);
 njs_int_t njs_regexp_prototype_exec(njs_vm_t *vm, njs_value_t *args,
     njs_uint_t nargs, njs_index_t unused);
 
diff -r c30a2a805014 -r f9082cd59ba6 src/njs_string.c
--- a/src/njs_string.c	Thu Jul 02 12:59:54 2020 +0000
+++ b/src/njs_string.c	Thu Jul 02 13:59:33 2020 +0000
@@ -3022,7 +3022,7 @@ njs_string_prototype_search(njs_vm_t *vm
 
         if (njs_regex_is_valid(&pattern->regex[n])) {
             ret = njs_regexp_match(vm, &pattern->regex[n], string.start,
-                                   string.size, vm->single_match_data);
+                                   0, string.size, vm->single_match_data);
             if (ret >= 0) {
                 captures = njs_regex_captures(vm->single_match_data);
                 index = njs_string_index(&string, captures[0]);
@@ -3147,7 +3147,7 @@ njs_string_match_multiple(njs_vm_t *vm, 
         end = p + string.size;
 
         do {
-            ret = njs_regexp_match(vm, &pattern->regex[type], p, string.size,
+            ret = njs_regexp_match(vm, &pattern->regex[type], p, 0, string.size,
                                    vm->single_match_data);
             if (ret < 0) {
                 if (njs_fast_path(ret == NJS_REGEX_NOMATCH)) {
@@ -3286,7 +3286,7 @@ njs_string_prototype_split(njs_vm_t *vm,
             end = string.start + string.size;
 
             do {
-                ret = njs_regexp_match(vm, &pattern->regex[type], start,
+                ret = njs_regexp_match(vm, &pattern->regex[type], start, 0,
                                        end - start, vm->single_match_data);
                 if (ret >= 0) {
                     captures = njs_regex_captures(vm->single_match_data);
@@ -3564,7 +3564,7 @@ njs_string_replace_regexp(njs_vm_t *vm, 
 
     do {
         ret = njs_regexp_match(vm, &pattern->regex[r->type],
-                               r->part[0].start, r->part[0].size,
+                               r->part[0].start, 0, r->part[0].size,
                                r->match_data);
 
         if (ret < 0) {
diff -r c30a2a805014 -r f9082cd59ba6 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Thu Jul 02 12:59:54 2020 +0000
+++ b/src/test/njs_unit_test.c	Thu Jul 02 13:59:33 2020 +0000
@@ -9711,6 +9711,9 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("var s; var r = /./g; while (s = r.exec('abc')); s"),
       njs_str("null") },
 
+    { njs_str("(/?/).exec('????').index"),
+      njs_str("1") },
+
     { njs_str("var r = /LS/i.exec(false); r[0]"),
       njs_str("ls") },
 

From xeioex at nginx.com  Thu Jul  2 14:01:44 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 02 Jul 2020 14:01:44 +0000
Subject: [njs] Fixed String.prototype.replace() according to the specification.
Message-ID: <hg.1c729f765cfb.1593698504.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/1c729f765cfb
branches:  
changeset: 1448:1c729f765cfb
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu Jul 02 14:00:16 2020 +0000
description:
Fixed String.prototype.replace() according to the specification.

This closes #308 issue on Github.

diffstat:

 src/njs_regexp.c         |   316 +++++++++++++
 src/njs_string.c         |  1055 ++++++++++-----------------------------------
 src/njs_string.h         |     3 +
 src/njs_value.c          |    25 +
 src/njs_value.h          |     3 +
 src/test/njs_unit_test.c |   365 ++++++++++-----
 6 files changed, 820 insertions(+), 947 deletions(-)

diffs (truncated from 1960 to 1000 lines):

diff -r f9082cd59ba6 -r 1c729f765cfb src/njs_regexp.c
--- a/src/njs_regexp.c	Thu Jul 02 13:59:33 2020 +0000
+++ b/src/njs_regexp.c	Thu Jul 02 14:00:16 2020 +0000
@@ -1161,6 +1161,314 @@ njs_regexp_string_create(njs_vm_t *vm, n
 }
 
 
+static njs_int_t
+njs_regexp_prototype_symbol_replace(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused)
+{
+    u_char             *p;
+    int64_t            n, last_index, ncaptures, pos, next_pos, size, length;
+    njs_str_t          rep, m;
+    njs_int_t          ret;
+    njs_arr_t          results;
+    njs_chb_t          chain;
+    njs_uint_t         i;
+    njs_bool_t         global;
+    njs_array_t        *array;
+    njs_value_t        *arguments, *r, *rx, *string, *replace;
+    njs_value_t        s_lvalue, r_lvalue, value, matched, groups, retval;
+    njs_function_t     *func_replace;
+    njs_string_prop_t  s;
+
+    static const njs_value_t  string_global = njs_string("global");
+    static const njs_value_t  string_groups = njs_string("groups");
+    static const njs_value_t  string_index = njs_string("index");
+    static const njs_value_t  string_lindex = njs_string("lastIndex");
+
+    rx = njs_argument(args, 0);
+
+    if (njs_slow_path(!njs_is_object(rx))) {
+        njs_type_error(vm, "\"this\" is not object");
+        return NJS_ERROR;
+    }
+
+    string = njs_lvalue_arg(&s_lvalue, args, nargs, 1);
+
+    ret = njs_value_to_string(vm, string, string);
+    if (njs_slow_path(ret != NJS_OK)) {
+        return ret;
+    }
+
+    length = njs_string_prop(&s, string);
+
+    rep.start = NULL;
+    rep.length = 0;
+
+    replace = njs_lvalue_arg(&r_lvalue, args, nargs, 2);
+    func_replace = njs_is_function(replace) ? njs_function(replace) : NULL;
+
+    if (!func_replace) {
+        ret = njs_value_to_string(vm, replace, replace);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+    }
+
+    ret = njs_value_property(vm, rx, njs_value_arg(&string_global), &value);
+    if (njs_slow_path(ret == NJS_ERROR)) {
+        return NJS_ERROR;
+    }
+
+    global = njs_bool(&value);
+
+    if (global) {
+        njs_set_number(&value, 0);
+        ret = njs_value_property_set(vm, rx, njs_value_arg(&string_lindex),
+                                     &value);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return NJS_ERROR;
+        }
+    }
+
+    njs_chb_init(&chain, vm->mem_pool);
+
+    results.separate = 0;
+    results.pointer = 0;
+
+    r = njs_arr_init(vm->mem_pool, &results, NULL, 4, sizeof(njs_value_t));
+    if (njs_slow_path(r == NULL)) {
+        return NJS_ERROR;
+    }
+
+    for ( ;; ) {
+        r = njs_arr_add(&results);
+        if (njs_slow_path(r == NULL)) {
+            ret = NJS_ERROR;
+            goto exception;
+        }
+
+        ret = njs_regexp_exec(vm, rx, string, r);
+        if (njs_slow_path(ret != NJS_OK)) {
+            goto exception;
+        }
+
+        if (njs_is_null(r) || !global) {
+            break;
+        }
+
+        if (njs_fast_path(njs_is_fast_array(r) && njs_array_len(r) != 0)) {
+            value = njs_array_start(r)[0];
+
+        } else {
+            ret = njs_value_property_i64(vm, r, 0, &value);
+            if (njs_slow_path(ret == NJS_ERROR)) {
+                goto exception;
+            }
+        }
+
+        ret = njs_value_to_string(vm, &value, &value);
+        if (njs_slow_path(ret != NJS_OK)) {
+            goto exception;
+        }
+
+        if (njs_string_length(&value) != 0) {
+            continue;
+        }
+
+        ret = njs_value_property(vm, rx, njs_value_arg(&string_lindex), &value);
+        if (njs_slow_path(ret == NJS_ERROR)) {
+            goto exception;
+        }
+
+        ret = njs_value_to_length(vm, &value, &last_index);
+        if (njs_slow_path(ret != NJS_OK)) {
+            goto exception;
+        }
+
+        njs_set_number(&value, last_index + 1);
+        ret = njs_value_property_set(vm, rx, njs_value_arg(&string_lindex),
+                                     &value);
+        if (njs_slow_path(ret != NJS_OK)) {
+            goto exception;
+        }
+    }
+
+    i = 0;
+    next_pos = 0;
+
+    while (i < results.items) {
+        r = njs_arr_item(&results, i++);
+
+        if (njs_slow_path(njs_is_null(r))) {
+            break;
+        }
+
+        ret = njs_value_property_i64(vm, r, 0, &matched);
+        if (njs_slow_path(ret == NJS_ERROR)) {
+            goto exception;
+        }
+
+        ret = njs_value_to_string(vm, &matched, &matched);
+        if (njs_slow_path(ret != NJS_OK)) {
+            goto exception;
+        }
+
+        ret = njs_value_property(vm, r, njs_value_arg(&string_index), &value);
+        if (njs_slow_path(ret == NJS_ERROR)) {
+            goto exception;
+        }
+
+        ret = njs_value_to_integer(vm, &value, &pos);
+        if (njs_slow_path(ret != NJS_OK)) {
+            goto exception;
+        }
+
+        if ((size_t) length != s.size) {
+            /* UTF-8 string. */
+            pos = njs_string_offset(s.start, s.start + s.size, pos) - s.start;
+        }
+
+        pos = njs_max(njs_min(pos, (int64_t) s.size), 0);
+
+        if (njs_fast_path(njs_is_fast_array(r))) {
+            array = njs_array(r);
+
+            arguments = array->start;
+            ncaptures = array->length;
+
+            for (n = 1; n < ncaptures; n++) {
+                if (njs_is_undefined(&arguments[n])) {
+                    continue;
+                }
+
+                ret = njs_value_to_string(vm, &arguments[n], &arguments[n]);
+                if (njs_slow_path(ret == NJS_ERROR)) {
+                    goto exception;
+                }
+            }
+
+        } else {
+            ret = njs_object_length(vm, r, &ncaptures);
+            if (njs_slow_path(ret != NJS_OK)) {
+                goto exception;
+            }
+
+            array = njs_array_alloc(vm, 0, ncaptures, 0);
+            if (njs_slow_path(array == NULL)) {
+                goto exception;
+            }
+
+            arguments = array->start;
+            arguments[0] = matched;
+
+            for (n = 1; n < ncaptures; n++) {
+                ret = njs_value_property_i64(vm, r, n, &arguments[n]);
+                if (njs_slow_path(ret == NJS_ERROR)) {
+                    goto exception;
+                }
+
+                if (njs_is_undefined(&arguments[n])) {
+                    continue;
+                }
+
+                ret = njs_value_to_string(vm, &arguments[n], &arguments[n]);
+                if (njs_slow_path(ret == NJS_ERROR)) {
+                    goto exception;
+                }
+            }
+        }
+
+        ret = njs_value_property(vm, r, njs_value_arg(&string_groups), &groups);
+        if (njs_slow_path(ret == NJS_ERROR)) {
+            goto exception;
+        }
+
+        if (!func_replace) {
+            if (njs_is_defined(&groups)) {
+                ret = njs_value_to_object(vm, &groups);
+                if (njs_slow_path(ret != NJS_OK)) {
+                    return ret;
+                }
+            }
+
+            ret = njs_string_get_substitution(vm, &matched, string, pos,
+                                              arguments, ncaptures - 1, &groups,
+                                              replace, &retval);
+
+        } else {
+            ret = njs_array_expand(vm, array, 0,
+                                   njs_is_defined(&groups) ? 3 : 2);
+            if (njs_slow_path(ret != NJS_OK)) {
+                goto exception;
+            }
+
+            arguments = array->start;
+            njs_set_number(&arguments[n++], pos);
+            arguments[n++] = *string;
+
+            if (njs_is_defined(&groups)) {
+                arguments[n++] = groups;
+            }
+
+            ret = njs_function_call(vm, func_replace,
+                                    njs_value_arg(&njs_value_undefined),
+                                    arguments, n, &retval);
+        }
+
+        if (njs_slow_path(ret == NJS_ERROR)) {
+            return NJS_ERROR;
+        }
+
+        ret = njs_value_to_string(vm, &retval, &retval);
+        if (njs_slow_path(ret != NJS_OK)) {
+            goto exception;
+        }
+
+        if (pos >= next_pos) {
+            njs_chb_append(&chain, &s.start[next_pos], pos - next_pos);
+
+            njs_string_get(&retval, &rep);
+            njs_chb_append_str(&chain, &rep);
+
+            njs_string_get(&matched, &m);
+
+            next_pos = pos + (int64_t) m.length;
+        }
+    }
+
+    if (next_pos < (int64_t) s.size) {
+        njs_chb_append(&chain, &s.start[next_pos], s.size - next_pos);
+    }
+
+    size = njs_chb_size(&chain);
+    if (njs_slow_path(size < 0)) {
+        njs_memory_error(vm);
+        ret = NJS_ERROR;
+        goto exception;
+    }
+
+    length = njs_chb_utf8_length(&chain);
+
+    p = njs_string_alloc(vm, &vm->retval, size, length);
+    if (njs_slow_path(p == NULL)) {
+        ret = NJS_ERROR;
+        goto exception;
+    }
+
+    njs_chb_join_to(&chain, p);
+
+    ret = NJS_OK;
+
+exception:
+
+    njs_chb_destroy(&chain);
+    njs_arr_destroy(&results);
+
+    return ret;
+}
+
+
+
+
 static const njs_object_prop_t  njs_regexp_constructor_properties[] =
 {
     {
@@ -1252,6 +1560,14 @@ static const njs_object_prop_t  njs_rege
         .writable = 1,
         .configurable = 1,
     },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_wellknown_symbol(NJS_SYMBOL_REPLACE),
+        .value = njs_native_function(njs_regexp_prototype_symbol_replace, 2),
+        .writable = 1,
+        .configurable = 1,
+    },
 };
 
 
diff -r f9082cd59ba6 -r 1c729f765cfb src/njs_string.c
--- a/src/njs_string.c	Thu Jul 02 13:59:33 2020 +0000
+++ b/src/njs_string.c	Thu Jul 02 14:00:16 2020 +0000
@@ -12,44 +12,6 @@
 #define NJS_TRIM_END    2
 
 
-typedef struct {
-    u_char                     *start;
-    size_t                     size;
-    njs_value_t                value;
-} njs_string_replace_part_t;
-
-
-#define NJS_SUBST_COPY        255
-#define NJS_SUBST_PRECEDING   254
-#define NJS_SUBST_FOLLOWING   253
-
-
-typedef struct {
-     uint32_t  type;
-     uint32_t  size;
-     u_char    *start;
-} njs_string_subst_t;
-
-
-typedef struct {
-    njs_value_t                retval;
-
-    njs_arr_t                  parts;
-    njs_string_replace_part_t  array[3];
-    njs_string_replace_part_t  *part;
-
-    njs_arr_t                  *substitutions;
-    njs_function_t             *function;
-
-    njs_regex_match_data_t     *match_data;
-
-    njs_bool_t                 empty;
-
-    njs_utf8_t                 utf8:8;
-    njs_regexp_utf8_t          type:8;
-} njs_string_replace_t;
-
-
 static void njs_encode_base64_core(njs_str_t *dst, const njs_str_t *src,
     const u_char *basis, njs_uint_t padding);
 static njs_int_t njs_decode_base64_core(njs_vm_t *vm,
@@ -72,25 +34,6 @@ static njs_int_t njs_string_match_multip
     njs_regexp_pattern_t *pattern);
 static njs_int_t njs_string_split_part_add(njs_vm_t *vm, njs_array_t *array,
     njs_utf8_t utf8, const u_char *start, size_t size);
-static njs_int_t njs_string_replace_regexp(njs_vm_t *vm, njs_value_t *this,
-    njs_value_t *regex, njs_string_replace_t *r);
-static njs_int_t njs_string_replace_regexp_function(njs_vm_t *vm,
-    njs_value_t *this, njs_value_t *regex, njs_string_replace_t *r,
-    int *captures, njs_uint_t n);
-static njs_int_t njs_string_replace_regexp_join(njs_vm_t *vm,
-    njs_string_replace_t *r);
-static njs_int_t njs_string_replace_search(njs_vm_t *vm, njs_value_t *this,
-    njs_value_t *search, njs_string_replace_t *r);
-static njs_int_t njs_string_replace_search_function(njs_vm_t *vm,
-        njs_value_t *this, njs_value_t *search, njs_string_replace_t *r);
-static njs_int_t njs_string_replace_parse(njs_vm_t *vm,
-    njs_string_replace_t *r, u_char *p, u_char *end, size_t size,
-    njs_uint_t ncaptures);
-static njs_int_t njs_string_replace_substitute(njs_vm_t *vm,
-    njs_string_replace_t *r, int *captures);
-static njs_int_t njs_string_replace_join(njs_vm_t *vm, njs_string_replace_t *r);
-static void njs_string_replacement_copy(njs_string_replace_part_t *string,
-    const njs_value_t *value);
 
 
 #define njs_base64_encoded_length(len)  (((len + 2) / 3) * 4)
@@ -3406,806 +3349,294 @@ njs_string_split_part_add(njs_vm_t *vm, 
 }
 
 
-static njs_int_t
-njs_string_prototype_replace(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
-    njs_index_t unused)
+njs_int_t
+njs_string_get_substitution(njs_vm_t *vm, njs_value_t *matched,
+    njs_value_t *string, int64_t pos, njs_value_t *captures, int64_t ncaptures,
+    njs_value_t *groups, njs_value_t *replacement, njs_value_t *retval)
 {
-    u_char                *p, *start, *end;
-    njs_int_t             ret;
-    njs_uint_t            ncaptures;
-    njs_value_t           *this, *search, *replace;
-    njs_value_t           search_lvalue, replace_lvalue;
-    njs_regex_t           *regex;
-    njs_string_prop_t     string;
-    njs_string_replace_t  *r, string_replace;
-
-    ret = njs_string_object_validate(vm, njs_arg(args, nargs, 0));
-    if (njs_slow_path(ret != NJS_OK)) {
-        return ret;
-    }
-
-    this = njs_argument(args, 0);
-
-    if (nargs == 1) {
-        goto original;
-    }
-
-    search = njs_lvalue_arg(&search_lvalue, args, nargs, 1);
-    replace = njs_lvalue_arg(&replace_lvalue, args, nargs, 2);
-
-    (void) njs_string_prop(&string, this);
-
-    if (string.size == 0) {
-        goto original;
-    }
-
-    r = &string_replace;
-
-    r->utf8 = NJS_STRING_BYTE;
-    r->type = NJS_REGEXP_BYTE;
-
-    if (string.length != 0) {
-        r->utf8 = NJS_STRING_ASCII;
-        r->type = NJS_REGEXP_UTF8;
-
-        if (string.length != string.size) {
-            r->utf8 = NJS_STRING_UTF8;
-        }
-    }
-
-    if (njs_is_regexp(search)) {
-        regex = &njs_regexp_pattern(search)->regex[r->type];
-
-        if (!njs_regex_is_valid(regex)) {
-            goto original;
+    int64_t      tail, size, length, n;
+    u_char       c, c2, *p, *r, *end;
+    njs_str_t    rep, m, str, cap;
+    njs_int_t    ret;
+    njs_chb_t    chain;
+    njs_value_t  name, value;
+
+    njs_string_get(replacement, &rep);
+    p = rep.start;
+    end = rep.start + rep.length;
+
+    njs_chb_init(&chain, vm->mem_pool);
+
+    while (p < end) {
+        r = njs_strlchr(p, end, '$');
+        if (r == NULL || r == &end[-1]) {
+            if (njs_fast_path(p == rep.start)) {
+                *retval = *replacement;
+                return NJS_OK;
+            }
+
+            njs_chb_append(&chain, p, end - p);
+            goto done;
         }
 
-        ncaptures = njs_regex_ncaptures(regex);
-
-    } else {
-        regex = NULL;
-        ncaptures = 1;
-
-        if (!njs_is_string(search)) {
-            ret = njs_value_to_string(vm, search, search);
-            if (njs_slow_path(ret != NJS_OK)) {
-                return ret;
+        njs_chb_append(&chain, p, r - p);
+        p = r;
+
+        c = r[1];
+
+        switch (c) {
+        case '$':
+            njs_chb_append_literal(&chain, "$");
+            p += 2;
+            break;
+
+        case '&':
+            njs_string_get(matched, &m);
+            njs_chb_append_str(&chain, &m);
+            p += 2;
+            break;
+
+        case '`':
+            njs_string_get(string, &str);
+            njs_chb_append(&chain, str.start, pos);
+            p += 2;
+            break;
+
+        case '\'':
+            njs_string_get(matched, &m);
+            tail = pos + m.length;
+
+            njs_string_get(string, &str);
+            njs_chb_append(&chain, &str.start[tail],
+                           njs_max((int64_t) str.length - tail, 0));
+            p += 2;
+            break;
+
+        case '<':
+            r = njs_strlchr(p, end, '>');
+            if (r == NULL) {
+                njs_chb_append(&chain, p, 2);
+                p += 2;
+                break;
+            }
+
+            p += 2;
+
+            if (groups == NULL) {
+                break;
             }
-        }
-    }
-
-    /* This cannot fail. */
-    r->part = njs_arr_init(vm->mem_pool, &r->parts, &r->array,
-                           3, sizeof(njs_string_replace_part_t));
-
-    r->substitutions = NULL;
-    r->function = NULL;
-
-    /* A literal replacement is stored in the second part. */
-
-    if (nargs == 2) {
-        njs_string_replacement_copy(&r->part[1], &njs_string_undefined);
-
-    } else if (njs_is_function(replace)) {
-        r->function = njs_function(replace);
-
-    } else {
-        if (njs_slow_path(!njs_is_string(replace))) {
-            ret = njs_value_to_string(vm, replace, replace);
+
+            ret = njs_vm_value_string_set(vm, &name, p, r - p);
             if (njs_slow_path(ret != NJS_OK)) {
-                return ret;
+                goto exception;
+            }
+
+            p = r + 1;
+
+            ret = njs_value_property(vm, groups, &name, &value);
+            if (njs_slow_path(ret == NJS_ERROR)) {
+                goto exception;
+            }
+
+            if (njs_is_defined(&value)) {
+                ret = njs_value_to_string(vm, &value, &value);
+                if (njs_slow_path(ret == NJS_ERROR)) {
+                    goto exception;
+                }
+
+                njs_string_get(&value, &str);
+                njs_chb_append_str(&chain, &str);
             }
-        }
-
-        njs_string_replacement_copy(&r->part[1], replace);
-
-        start = r->part[1].start;
-
-        if (start == NULL) {
-            start = r->part[1].value.short_string.start;
-        }
-
-        end = start + r->part[1].size;
-
-        for (p = start; p < end; p++) {
-            if (*p == '$') {
-                ret = njs_string_replace_parse(vm, r, p, end, p - start,
-                                               ncaptures);
-                if (njs_slow_path(ret != NJS_OK)) {
-                    return ret;
+
+            break;
+
+        default:
+            if (c >= '0' && c <= '9') {
+                n = c - '0';
+
+                c2 = (&r[2] < end) ? r[2] : 0;
+
+                if (c2 >= '0' && c2 <= '9'
+                    && (n * 10 + (c2 - '0')) <= ncaptures)
+                {
+                    n = n * 10 + (c2 - '0');
+
+                } else {
+                    c2 = 0;
                 }
 
-                /* Reset parts array to the subject string only. */
-                r->parts.items = 1;
+                if (n == 0 || n > ncaptures) {
+                    njs_chb_append(&chain, p, (c2 != 0) ? 3 : 2);
+                    p += (c2 != 0) ? 3 : 2;
+                    break;
+                }
+
+                p += (c2 != 0) ? 3 : 2;
+
+                if (njs_is_defined(&captures[n])) {
+                    njs_string_get(&captures[n], &cap);
+                    njs_chb_append_str(&chain, &cap);
+                }
 
                 break;
             }
-        }
-    }
-
-    r->part[0].start = string.start;
-    r->part[0].size = string.size;
-    njs_set_invalid(&r->part[0].value);
-
-    if (regex != NULL) {
-        r->match_data = njs_regex_match_data(regex, vm->regex_context);
-        if (njs_slow_path(r->match_data == NULL)) {
-            return NJS_ERROR;
-        }
-
-        return njs_string_replace_regexp(vm, this, search, r);
-    }
-
-    return njs_string_replace_search(vm, this, search, r);
-
-original:
-
-    njs_string_copy(&vm->retval, this);
-
-    return NJS_OK;
-}
-
-
-static njs_int_t
-njs_string_replace_regexp(njs_vm_t *vm, njs_value_t *this, njs_value_t *regex,
-    njs_string_replace_t *r)
-{
-    int                        *captures;
-    u_char                     *p, *start;
-    njs_int_t                  ret;
-    const u_char               *end;
-    njs_regexp_pattern_t       *pattern;
-    njs_string_replace_part_t  replace;
-
-    pattern = njs_regexp_pattern(regex);
-    end = r->part[0].start + r->part[0].size;
-
-    replace = r->part[1];
-
-    do {
-        ret = njs_regexp_match(vm, &pattern->regex[r->type],
-                               r->part[0].start, 0, r->part[0].size,
-                               r->match_data);
-
-        if (ret < 0) {
-            if (njs_slow_path(ret != NJS_REGEX_NOMATCH)) {
-                return NJS_ERROR;
-            }
-
+
+            njs_chb_append_literal(&chain, "$");
+            p += 1;
             break;
         }
-
-        captures = njs_regex_captures(r->match_data);
-
-        if (r->substitutions != NULL) {
-            ret = njs_string_replace_substitute(vm, r, captures);
-            if (njs_slow_path(ret != NJS_OK)) {
-                return ret;
-            }
-
-            if (!pattern->global) {
-                return njs_string_replace_regexp_join(vm, r);
-            }
-
-            continue;
-        }
-
-        if (r->part != r->parts.start) {
-            r->part = njs_arr_add(&r->parts);
-            if (njs_slow_path(r->part == NULL)) {
-                return NJS_ERROR;
-            }
-
-            r->part = njs_arr_add(&r->parts);
-            if (njs_slow_path(r->part == NULL)) {
-                return NJS_ERROR;
-            }
-
-            r->part -= 2;
-        }
-
-        if (captures[1] == 0) {
-
-            /* Empty match. */
-
-            start = r->part[0].start;
-
-            if (start < end) {
-                p = (r->utf8 != NJS_STRING_BYTE)
-                    ? (u_char *) njs_utf8_next(start, end) : start + 1;
-
-                r->part[1].start = start;
-                r->part[1].size = p - start;
-
-                r->part[2].start = p;
-                r->part[2].size = end - p;
-
-            } else {
-                r->part[1].size = 0;
-                r->part[2].size = 0;
-
-                /* To exit the loop. */
-                r->part[2].start = start + 1;
-            }
-
-            if (r->function != NULL) {
-                return njs_string_replace_regexp_function(vm, this, regex, r,
-                                                          captures, ret);
-            }
-
-            r->part[0] = replace;
-
-        } else {
-            r->part[2].start = r->part[0].start + captures[1];
-            r->part[2].size = r->part[0].size - captures[1];
-            njs_set_invalid(&r->part[2].value);
-
-            if (r->function != NULL) {
-                return njs_string_replace_regexp_function(vm, this, regex, r,
-                                                          captures, ret);
-            }
-
-            r->part[0].size = captures[0];
-
-            r->part[1] = replace;
-        }
-
-        if (!pattern->global) {
-            return njs_string_replace_regexp_join(vm, r);
-        }
-
-        r->part += 2;
-
-    } while (r->part[0].start <= end);
-
-    if (r->part != r->parts.start) {
-        return njs_string_replace_regexp_join(vm, r);
+    }
+
+done:
+
+    size = njs_chb_size(&chain);
+    if (njs_slow_path(size < 0)) {
+        njs_memory_error(vm);
+        ret = NJS_ERROR;
+        goto exception;
     }
 
-    njs_regex_match_data_free(r->match_data, vm->regex_context);
-
-    njs_arr_destroy(&r->parts);
-
-    njs_string_copy(&vm->retval, this);
+    length = njs_chb_utf8_length(&chain);
+
+    p = njs_string_alloc(vm, retval, size, length);
+    if (njs_slow_path(p == NULL)) {
+        ret = NJS_ERROR;
+        goto exception;
+    }
+
+    njs_chb_join_to(&chain, p);
+
+    ret = NJS_OK;
+
+exception:
+
+    njs_chb_destroy(&chain);
 
     return NJS_OK;
 }
 
 
 static njs_int_t
-njs_string_replace_regexp_function(njs_vm_t *vm, njs_value_t *this,
-    njs_value_t *regex, njs_string_replace_t *r, int *captures, njs_uint_t n)
+njs_string_prototype_replace(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
 {
-    u_char             *start;
-    size_t             size, length;
+    u_char             *r;
+    size_t             length, search_length, ret_length, size;
+    int64_t            pos;
     njs_int_t          ret;
-    njs_uint_t         i, k;
-    njs_value_t        *arguments;
-    njs_string_prop_t  string;
-
-    if (njs_slow_path((n + 3) >= UINT32_MAX / sizeof(njs_value_t))) {
-        njs_memory_error(vm);
+    njs_value_t        *this, *search, *replace;
+    njs_value_t        search_lvalue, replace_lvalue, replacer, retval,
+                       arguments[3];
+    const u_char       *p;
+    njs_function_t     *func_replace;
+    njs_string_prop_t  string, s, ret_string;
+
+    static const njs_value_t  replace_key =
+                                      njs_wellknown_symbol(NJS_SYMBOL_REPLACE);
+
+    this = njs_argument(args, 0);
+
+    if (njs_slow_path(njs_is_null_or_undefined(this))) {
+        njs_type_error(vm, "cannot convert \"%s\"to object",
+                       njs_type_string(this->type));
         return NJS_ERROR;
     }
 
-    njs_set_invalid(&r->retval);
-
-    arguments = njs_mp_alloc(vm->mem_pool, (n + 3) * sizeof(njs_value_t));
-    if (njs_slow_path(arguments == NULL)) {
-        njs_memory_error(vm);
-        return NJS_ERROR;
+    search = njs_lvalue_arg(&search_lvalue, args, nargs, 1);
+    replace = njs_lvalue_arg(&replace_lvalue, args, nargs, 2);
+
+    if (!njs_is_null_or_undefined(search)) {
+        ret = njs_value_method(vm, search, njs_value_arg(&replace_key),
+                               &replacer);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        if (njs_is_defined(&replacer)) {
+            arguments[0] = *this;
+            arguments[1] = *replace;
+
+            return njs_function_call(vm, njs_function(&replacer), search,
+                                     arguments, 2, &vm->retval);
+        }
+    }
+
+    ret = njs_value_to_string(vm, this, this);
+    if (njs_slow_path(ret != NJS_OK)) {
+        return ret;
+    }
+
+    ret = njs_value_to_string(vm, search, search);
+    if (njs_slow_path(ret != NJS_OK)) {
+        return ret;
     }
 
-    njs_set_undefined(&arguments[0]);
-
-    /* Matched substring and parenthesized submatch strings. */
-    for (k = 0, i = 1; i <= n; i++) {
-
-        start = r->part[0].start + captures[k];
-        size = captures[k + 1] - captures[k];
-        k += 2;
-
-        length = njs_string_calc_length(r->utf8, start, size);
-
-        ret = njs_string_new(vm, &arguments[i], start, size, length);
+    func_replace = njs_is_function(replace) ? njs_function(replace) : NULL;
+
+    if (func_replace == NULL) {
+        ret = njs_value_to_string(vm, replace, replace);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+    }
+
+    length = njs_string_prop(&string, this);
+    search_length = njs_string_prop(&s, search);
+
+    pos = njs_string_index_of(&string, &s, 0);
+    if (pos < 0) {
+        vm->retval = *this;
+        return NJS_OK;
+    }
+
+    if (func_replace == NULL) {
+        ret = njs_string_get_substitution(vm, search, this, pos, NULL, 0, NULL,
+                                          replace, &retval);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+    } else {
+        arguments[0] = *search;
+        njs_set_number(&arguments[1], pos);
+        arguments[2] = *this;
+
+        ret = njs_function_call(vm, func_replace,
+                                njs_value_arg(&njs_value_undefined),
+                                arguments, 3, &retval);
+
+        ret = njs_value_to_string(vm, &retval, &retval);
         if (njs_slow_path(ret != NJS_OK)) {
             return NJS_ERROR;
         }
     }
 
-    r->empty = (captures[0] == captures[1]);
-
-    /* The offset of the matched substring. */
-    njs_set_number(&arguments[n + 1], captures[0]);
-
-    /* The whole string being examined. */
-    length = njs_string_calc_length(r->utf8, r->part[0].start, r->part[0].size);
-
-    (void) njs_string_prop(&string, this);
-
-    ret = njs_string_new(vm, &arguments[n + 2], string.start, string.size,
-                         length);
-
-    if (njs_slow_path(ret != NJS_OK)) {
-        return NJS_ERROR;
-    }
-
-    r->part[0].size = captures[0];
-
-    ret = njs_function_apply(vm, r->function, arguments, n + 3, &r->retval);
-    if (njs_slow_path(ret != NJS_OK)) {
-        goto exception;
-    }
-
-    if (njs_slow_path(!njs_is_string(&r->retval))) {
-        ret = njs_value_to_string(vm, &r->retval, &r->retval);
-        if (njs_slow_path(ret != NJS_OK)) {
-            goto exception;
-        }
-    }
-
-    njs_string_replacement_copy(&r->part[r->empty ? 0 : 1], &r->retval);
-
-    if (njs_regexp_pattern(regex)->global) {
-        r->part += 2;
-
-        if (r->part[0].start > (string.start + string.size)) {
-            return njs_string_replace_regexp_join(vm, r);
-        }
-
-        return njs_string_replace_regexp(vm, this, regex, r);
-    }
-
-    return njs_string_replace_regexp_join(vm, r);
-
-exception:
-
-    njs_regex_match_data_free(r->match_data, vm->regex_context);
-
-    return NJS_ERROR;
-}
-
-

From weijunjie at didiglobal.com  Fri Jul  3 12:00:12 2020
From: weijunjie at didiglobal.com (=?gb2312?B?zrq/ob3cIEp1bmppZSBXZWk=?=)
Date: Fri, 3 Jul 2020 12:00:12 +0000
Subject: Thrift proxy code contribution
Message-ID: <1593777583821.6502@didiglobal.com>

Hello,

I am a developer from Beijing Xiaoju Technology Co, Ltd, China.?


Recently, we have make nginx(base on version 1.13.12) support thrift protocol proxing, following features are supported:

  1.  thrift proxing without idl (support thrift 0.11.0: socket transport/framed transport in strict write way)

  2.  tcp active health check

  3.  upstream keepalive connection

  4.  dynamic upstream

  5.  limiting request base on muti-variable

  6.  dispatching request to different upstream base on variable

  7.  load balance using swrr

Limitation:

  1.  thrift protocol:

     *   thrift protocol must support TMultiplexProtocol

     *   transport: socket transport, framed transport in strict write way

     *   protocol: binary protocol

  2.  event mechanics

     *   epoll

Status:

The program have been run online (CentOS 7.2,Linux version 3.10.0-514.16.1.el7.x86_64) for 3 months, and we are still making effort to optimize it to serving much more applications.


Are you willing to accept our code as a part of open source nginx?

Hope for your replay!

Thanks!

Junjie Wei

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200703/204b704c/attachment.htm>

From ru at nginx.com  Fri Jul  3 13:24:19 2020
From: ru at nginx.com (Ruslan Ermilov)
Date: Fri, 03 Jul 2020 13:24:19 +0000
Subject: [nginx] HTTP/2: lingering close after GOAWAY.
Message-ID: <hg.c5840ca2063d.1593782659.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/c5840ca2063d
branches:  
changeset: 7673:c5840ca2063d
user:      Ruslan Ermilov <ru at nginx.com>
date:      Fri Jul 03 16:16:47 2020 +0300
description:
HTTP/2: lingering close after GOAWAY.

After sending the GOAWAY frame, a connection is now closed using
the lingering close mechanism.

This allows for the reliable delivery of the GOAWAY frames, while
also fixing connection resets observed when http2_max_requests is
reached (ticket #1250), or with graceful shutdown (ticket #1544),
when some additional data from the client is received on a fully
closed connection.

For HTTP/2, the settings lingering_close, lingering_timeout, and
lingering_time are taken from the "server" level.

diffstat:

 src/http/v2/ngx_http_v2.c |  128 +++++++++++++++++++++++++++++++++++++++++++--
 src/http/v2/ngx_http_v2.h |    2 +
 2 files changed, 124 insertions(+), 6 deletions(-)

diffs (187 lines):

diff -r 3dcb1aba894a -r c5840ca2063d src/http/v2/ngx_http_v2.c
--- a/src/http/v2/ngx_http_v2.c	Mon Jun 29 17:15:51 2020 +0300
+++ b/src/http/v2/ngx_http_v2.c	Fri Jul 03 16:16:47 2020 +0300
@@ -60,6 +60,8 @@ typedef struct {
 static void ngx_http_v2_read_handler(ngx_event_t *rev);
 static void ngx_http_v2_write_handler(ngx_event_t *wev);
 static void ngx_http_v2_handle_connection(ngx_http_v2_connection_t *h2c);
+static void ngx_http_v2_lingering_close(ngx_http_v2_connection_t *h2c);
+static void ngx_http_v2_lingering_close_handler(ngx_event_t *rev);
 
 static u_char *ngx_http_v2_state_proxy_protocol(ngx_http_v2_connection_t *h2c,
     u_char *pos, u_char *end);
@@ -661,7 +663,7 @@ ngx_http_v2_handle_connection(ngx_http_v
     }
 
     if (h2c->goaway) {
-        ngx_http_close_connection(c);
+        ngx_http_v2_lingering_close(h2c);
         return;
     }
 
@@ -699,6 +701,113 @@ ngx_http_v2_handle_connection(ngx_http_v
 }
 
 
+static void
+ngx_http_v2_lingering_close(ngx_http_v2_connection_t *h2c)
+{
+    ngx_event_t               *rev, *wev;
+    ngx_connection_t          *c;
+    ngx_http_core_loc_conf_t  *clcf;
+
+    c = h2c->connection;
+
+    clcf = ngx_http_get_module_loc_conf(h2c->http_connection->conf_ctx,
+                                        ngx_http_core_module);
+
+    if (clcf->lingering_close == NGX_HTTP_LINGERING_OFF) {
+        ngx_http_close_connection(c);
+        return;
+    }
+
+    rev = c->read;
+    rev->handler = ngx_http_v2_lingering_close_handler;
+
+    h2c->lingering_time = ngx_time() + (time_t) (clcf->lingering_time / 1000);
+    ngx_add_timer(rev, clcf->lingering_timeout);
+
+    if (ngx_handle_read_event(rev, 0) != NGX_OK) {
+        ngx_http_close_connection(c);
+        return;
+    }
+
+    wev = c->write;
+    wev->handler = ngx_http_empty_handler;
+
+    if (wev->active && (ngx_event_flags & NGX_USE_LEVEL_EVENT)) {
+        if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) != NGX_OK) {
+            ngx_http_close_connection(c);
+            return;
+        }
+    }
+
+    if (ngx_shutdown_socket(c->fd, NGX_WRITE_SHUTDOWN) == -1) {
+        ngx_connection_error(c, ngx_socket_errno,
+                             ngx_shutdown_socket_n " failed");
+        ngx_http_close_connection(c);
+        return;
+    }
+
+    if (rev->ready) {
+        ngx_http_v2_lingering_close_handler(rev);
+    }
+}
+
+
+static void
+ngx_http_v2_lingering_close_handler(ngx_event_t *rev)
+{
+    ssize_t                    n;
+    ngx_msec_t                 timer;
+    ngx_connection_t          *c;
+    ngx_http_core_loc_conf_t  *clcf;
+    ngx_http_v2_connection_t  *h2c;
+    u_char                     buffer[NGX_HTTP_LINGERING_BUFFER_SIZE];
+
+    c = rev->data;
+    h2c = c->data;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                   "http2 lingering close handler");
+
+    if (rev->timedout) {
+        ngx_http_close_connection(c);
+        return;
+    }
+
+    timer = (ngx_msec_t) h2c->lingering_time - (ngx_msec_t) ngx_time();
+    if ((ngx_msec_int_t) timer <= 0) {
+        ngx_http_close_connection(c);
+        return;
+    }
+
+    do {
+        n = c->recv(c, buffer, NGX_HTTP_LINGERING_BUFFER_SIZE);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "lingering read: %z", n);
+
+        if (n == NGX_ERROR || n == 0) {
+            ngx_http_close_connection(c);
+            return;
+        }
+
+    } while (rev->ready);
+
+    if (ngx_handle_read_event(rev, 0) != NGX_OK) {
+        ngx_http_close_connection(c);
+        return;
+    }
+
+    clcf = ngx_http_get_module_loc_conf(h2c->http_connection->conf_ctx,
+                                        ngx_http_core_module);
+    timer *= 1000;
+
+    if (timer > clcf->lingering_timeout) {
+        timer = clcf->lingering_timeout;
+    }
+
+    ngx_add_timer(rev, timer);
+}
+
+
 static u_char *
 ngx_http_v2_state_proxy_protocol(ngx_http_v2_connection_t *h2c, u_char *pos,
     u_char *end)
@@ -4541,16 +4650,15 @@ ngx_http_v2_finalize_connection(ngx_http
     h2c->blocked = 1;
 
     if (!c->error && !h2c->goaway) {
+        h2c->goaway = 1;
+
         if (ngx_http_v2_send_goaway(h2c, status) != NGX_ERROR) {
             (void) ngx_http_v2_send_output_queue(h2c);
         }
     }
 
-    c->error = 1;
-
     if (!h2c->processing && !h2c->pushing) {
-        ngx_http_close_connection(c);
-        return;
+        goto done;
     }
 
     c->read->handler = ngx_http_empty_handler;
@@ -4598,10 +4706,18 @@ ngx_http_v2_finalize_connection(ngx_http
     h2c->blocked = 0;
 
     if (h2c->processing || h2c->pushing) {
+        c->error = 1;
         return;
     }
 
-    ngx_http_close_connection(c);
+done:
+
+    if (c->error) {
+        ngx_http_close_connection(c);
+        return;
+    }
+
+    ngx_http_v2_lingering_close(h2c);
 }
 
 
diff -r 3dcb1aba894a -r c5840ca2063d src/http/v2/ngx_http_v2.h
--- a/src/http/v2/ngx_http_v2.h	Mon Jun 29 17:15:51 2020 +0300
+++ b/src/http/v2/ngx_http_v2.h	Fri Jul 03 16:16:47 2020 +0300
@@ -157,6 +157,8 @@ struct ngx_http_v2_connection_s {
     ngx_uint_t                       last_sid;
     ngx_uint_t                       last_push;
 
+    time_t                           lingering_time;
+
     unsigned                         closed_nodes:8;
     unsigned                         settings_ack:1;
     unsigned                         table_update:1;

From salmaanpehlari at gmail.com  Sun Jul  5 07:51:07 2020
From: salmaanpehlari at gmail.com (Salmaan Pehlari)
Date: Sun, 05 Jul 2020 00:51:07 -0700
Subject: [PATCH] SSL: Verify IP SAN's in upstream certificates
Message-ID: <3b843e88de3761b2b71b.1593935467@DESKTOP-L9O73KF.localdomain>

# HG changeset patch
# User Salmaan Pehlari <salmaanpehlari at gmail.com>
# Date 1593931168 25200
#      Sat Jul 04 23:39:28 2020 -0700
# Node ID 3b843e88de3761b2b71bac3c5fe453e09ae7990e
# Parent  c5840ca2063d26e432264ad0b0fe00c0bd94252c
SSL: Verify IP SAN's in upstream certificates.

Verify IP's in upstream certificates if no host names match.

diff -r c5840ca2063d -r 3b843e88de37 src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c	Fri Jul 03 16:16:47 2020 +0300
+++ b/src/event/ngx_event_openssl.c	Sat Jul 04 23:39:28 2020 -0700
@@ -4116,13 +4116,20 @@
     }
 
     if (X509_check_host(cert, (char *) name->data, name->len, 0, NULL) != 1) {
-        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                       "X509_check_host(): no match");
-        goto failed;
+
+        char *ip = (char *) ngx_palloc(c->pool, (name->len+1 * sizeof(char)));
+        ngx_memcpy(ip, name->data, name->len);
+        ip[name->len] = '\0';
+
+        if (X509_check_ip_asc(cert, ip, 0) != 1 {
+            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "X509_check_host() & X590_check_ip_asc: no match");
+            goto failed;
+        }
     }
 
     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                   "X509_check_host(): match");
+                   "X509_check_host() | X509_check_ip_asc: match");
 
     goto found;
 
@@ -4148,21 +4155,47 @@
         for (i = 0; i < n; i++) {
             altname = sk_GENERAL_NAME_value(altnames, i);
 
-            if (altname->type != GEN_DNS) {
-                continue;
-            }
-
-            str = altname->d.dNSName;
-
-            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                           "SSL subjectAltName: \"%*s\"",
-                           ASN1_STRING_length(str), ASN1_STRING_data(str));
-
-            if (ngx_ssl_check_name(name, str) == NGX_OK) {
-                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                               "SSL subjectAltName: match");
-                GENERAL_NAMES_free(altnames);
-                goto found;
+            if (altname->type == GEN_DNS) {
+
+                str = altname->d.dNSName;
+
+                ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                               "SSL subjectAltName: \"%*s\"",
+                               ASN1_STRING_length(str), ASN1_STRING_data(str));
+
+                if (ngx_ssl_check_name(name, str) == NGX_OK) {
+                    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                                   "SSL subjectAltName: match");
+                    GENERAL_NAMES_free(altnames);
+                    goto found;
+                }
+            } else if (altname->type == GEN_IPADD) {
+                x509_ip = altname->d.iPAddress;
+
+                if (x509_ip && x509_ip->data && x509_ip->length) {
+                    ip = (char *) ngx_palloc(c->pool, (name->len+1 * sizeof(char)));
+                    ngx_memcpy(ip, name->data, name->len);
+                    ip[name->len] = '\0';
+
+                    if (inet_pton(AF_INET, (const char *), ip, &(sa.sin_addr)) != 1) {
+                        if (inet_pton(AF_INET6, (const char *), ip, &(sa.sin_addr)) != 1) {
+                            GENERAL_NAME_free(altnames);
+                            goto failed;
+                        }
+                    }
+
+                    ip_octet = ASN1_OCTET_STRING_new();
+                    ASN1_STRING_set(ip_octet, &san.sin_addr, sizeof(sa.sinaddr));
+
+                    if (ASN1_STRING_cmp(x509_ip, ip_octet) == 0) {
+                        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL subjectAltName IP: match");
+                        ASN1_STRING_free(ip_octet);
+                        GENERAL_NAMES_free(altnames);
+                        goto found;
+                    }
+
+                    ASN1_STRING_free(ip_octet);
+                }
             }
         }
 

From salmaanpehlari at gmail.com  Sun Jul  5 08:09:07 2020
From: salmaanpehlari at gmail.com (Salmaan Pehlari)
Date: Sun, 5 Jul 2020 01:09:07 -0700
Subject: [PATCH] SSL: Verify IP SAN's in upstream certificates
Message-ID: <CAFFE-8R-8hbycAz0aFMPbP9Wr9ri91ZAPixeOfEfy_yZgt2Wyg@mail.gmail.com>

# HG changeset patch
# User Salmaan Pehlari <salmaanpehlari at gmail.com>
# Date 1593931168 25200
#      Sat Jul 04 23:39:28 2020 -0700
# Node ID 3b843e88de3761b2b71bac3c5fe453e09ae7990e
# Parent  c5840ca2063d26e432264ad0b0fe00c0bd94252c
SSL: Verify IP SAN's in upstream certificates.

Verify IP's in upstream certificates if no host names match.

diff -r c5840ca2063d -r 3b843e88de37 src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c     Fri Jul 03 16:16:47 2020 +0300
+++ b/src/event/ngx_event_openssl.c     Sat Jul 04 23:39:28 2020 -0700
@@ -4116,13 +4116,20 @@
     }

     if (X509_check_host(cert, (char *) name->data, name->len, 0, NULL) !=
1) {
-        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                       "X509_check_host(): no match");
-        goto failed;
+
+        char *ip = (char *) ngx_palloc(c->pool, (name->len+1 *
sizeof(char)));
+        ngx_memcpy(ip, name->data, name->len);
+        ip[name->len] = '\0';
+
+        if (X509_check_ip_asc(cert, ip, 0) != 1 {
+            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "X509_check_host() & X590_check_ip_asc: no
match");
+            goto failed;
+        }
     }

     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                   "X509_check_host(): match");
+                   "X509_check_host() | X509_check_ip_asc: match");

     goto found;

@@ -4148,21 +4155,47 @@
         for (i = 0; i < n; i++) {
             altname = sk_GENERAL_NAME_value(altnames, i);

-            if (altname->type != GEN_DNS) {
-                continue;
-            }
-
-            str = altname->d.dNSName;
-
-            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                           "SSL subjectAltName: \"%*s\"",
-                           ASN1_STRING_length(str), ASN1_STRING_data(str));
-
-            if (ngx_ssl_check_name(name, str) == NGX_OK) {
-                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                               "SSL subjectAltName: match");
-                GENERAL_NAMES_free(altnames);
-                goto found;
+            if (altname->type == GEN_DNS) {
+
+                str = altname->d.dNSName;
+
+                ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                               "SSL subjectAltName: \"%*s\"",
+                               ASN1_STRING_length(str),
ASN1_STRING_data(str));
+
+                if (ngx_ssl_check_name(name, str) == NGX_OK) {
+                    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                                   "SSL subjectAltName: match");
+                    GENERAL_NAMES_free(altnames);
+                    goto found;
+                }
+            } else if (altname->type == GEN_IPADD) {
+                x509_ip = altname->d.iPAddress;
+
+                if (x509_ip && x509_ip->data && x509_ip->length) {
+                    ip = (char *) ngx_palloc(c->pool, (name->len+1 *
sizeof(char)));
+                    ngx_memcpy(ip, name->data, name->len);
+                    ip[name->len] = '\0';
+
+                    if (inet_pton(AF_INET, (const char *), ip,
&(sa.sin_addr)) != 1) {
+                        if (inet_pton(AF_INET6, (const char *), ip,
&(sa.sin_addr)) != 1) {
+                            GENERAL_NAME_free(altnames);
+                            goto failed;
+                        }
+                    }
+
+                    ip_octet = ASN1_OCTET_STRING_new();
+                    ASN1_STRING_set(ip_octet, &san.sin_addr,
sizeof(sa.sinaddr));
+
+                    if (ASN1_STRING_cmp(x509_ip, ip_octet) == 0) {
+                        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
"SSL subjectAltName IP: match");
+                        ASN1_STRING_free(ip_octet);
+                        GENERAL_NAMES_free(altnames);
+                        goto found;
+                    }
+
+                    ASN1_STRING_free(ip_octet);
+                }
             }
         }
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200705/cdc26377/attachment.htm>

From pluknet at nginx.com  Mon Jul  6 15:10:28 2020
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Mon, 6 Jul 2020 18:10:28 +0300
Subject: [nginx-test] Allow some syslog message to arrive in either order
In-Reply-To: <CAAH4kHZ1bjMO8fZvBVS2=2ov7_L9V4JL-43HpDxLVvYT=cD3Vg@mail.gmail.com>
References: <CAAH4kHZ1bjMO8fZvBVS2=2ov7_L9V4JL-43HpDxLVvYT=cD3Vg@mail.gmail.com>
Message-ID: <E4099498-F57F-49F1-BF0C-2BF87FD552F1@nginx.com>


> On 1 Jul 2020, at 02:09, Dionna Amalie Glaze <dionnaglaze at google.com> wrote:
> 
> # HG changeset patch
> # User Dionna Glaze <dionnaglaze at google.com>
> # Date 1593542175 25200
> #      Tue Jun 30 11:36:15 2020 -0700
> # Node ID a2a00a127689c64d21842d2aba7f1dc53e06b095
> # Parent  f55d25e08b3edaab6f3a30a000ac486334994978
> Allow some syslog message to arrive in either order
> 
> This test flakes occasionally with good,work arriving as work,good.

A slightly different version committed, thanks.
https://hg.nginx.org/nginx-tests/rev/9e142c0e34b2

-- 
Sergey Kandaurov


From mdounin at mdounin.ru  Mon Jul  6 17:59:10 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 6 Jul 2020 20:59:10 +0300
Subject: Thrift proxy code contribution
In-Reply-To: <1593777583821.6502@didiglobal.com>
References: <1593777583821.6502@didiglobal.com>
Message-ID: <20200706175910.GD12747@mdounin.ru>

Hello!

On Fri, Jul 03, 2020 at 12:00:12PM +0000, ??? Junjie Wei wrote:

[...]

> Are you willing to accept our code as a part of open source nginx?

Consider publishing your work as a module for nginx.

-- 
Maxim Dounin
http://mdounin.ru/

From mdounin at mdounin.ru  Mon Jul  6 18:00:51 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 06 Jul 2020 18:00:51 +0000
Subject: [nginx] Memcached: protect from too long responses.
Message-ID: <hg.7731c710796f.1594058451.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/7731c710796f
branches:  
changeset: 7674:7731c710796f
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Mon Jul 06 18:36:17 2020 +0300
description:
Memcached: protect from too long responses.

If a memcached response was followed by a correct trailer, and then
the NUL character followed by some extra data - this was accepted by
the trailer checking code.  This in turn resulted in ctx->rest underflow
and caused negative size buffer on the next reading from the upstream,
followed by the "negative size buf in writer" alert.

Fix is to always check for too long responses, so a correct trailer cannot
be followed by extra data.

diffstat:

 src/http/modules/ngx_http_memcached_module.c |  9 ++++++---
 1 files changed, 6 insertions(+), 3 deletions(-)

diffs (28 lines):

diff -r c5840ca2063d -r 7731c710796f src/http/modules/ngx_http_memcached_module.c
--- a/src/http/modules/ngx_http_memcached_module.c	Fri Jul 03 16:16:47 2020 +0300
+++ b/src/http/modules/ngx_http_memcached_module.c	Mon Jul 06 18:36:17 2020 +0300
@@ -485,10 +485,11 @@ ngx_http_memcached_filter(void *data, ss
 
     if (u->length == (ssize_t) ctx->rest) {
 
-        if (ngx_strncmp(b->last,
+        if (bytes > u->length
+            || ngx_strncmp(b->last,
                    ngx_http_memcached_end + NGX_HTTP_MEMCACHED_END - ctx->rest,
                    bytes)
-            != 0)
+               != 0)
         {
             ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0,
                           "memcached sent invalid trailer");
@@ -540,7 +541,9 @@ ngx_http_memcached_filter(void *data, ss
 
     last += (size_t) (u->length - NGX_HTTP_MEMCACHED_END);
 
-    if (ngx_strncmp(last, ngx_http_memcached_end, b->last - last) != 0) {
+    if (bytes > u->length
+        || ngx_strncmp(last, ngx_http_memcached_end, b->last - last) != 0)
+    {
         ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0,
                       "memcached sent invalid trailer");
 

From mdounin at mdounin.ru  Mon Jul  6 18:00:54 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 06 Jul 2020 18:00:54 +0000
Subject: [nginx] Proxy: drop extra data sent by upstream.
Message-ID: <hg.9afa45068b8f.1594058454.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/9afa45068b8f
branches:  
changeset: 7675:9afa45068b8f
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Mon Jul 06 18:36:19 2020 +0300
description:
Proxy: drop extra data sent by upstream.

Previous behaviour was to pass everything to the client, but this
seems to be suboptimal and causes issues (ticket #1695).  Fix is to
drop extra data instead, as it naturally happens in most clients.

diffstat:

 src/http/modules/ngx_http_proxy_module.c |  52 ++++++++++++++++++++++++++-----
 1 files changed, 43 insertions(+), 9 deletions(-)

diffs (81 lines):

diff -r 7731c710796f -r 9afa45068b8f src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c	Mon Jul 06 18:36:17 2020 +0300
+++ b/src/http/modules/ngx_http_proxy_module.c	Mon Jul 06 18:36:19 2020 +0300
@@ -2015,6 +2015,25 @@ ngx_http_proxy_copy_filter(ngx_event_pip
         return NGX_OK;
     }
 
+    if (p->upstream_done) {
+        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, p->log, 0,
+                       "http proxy data after close");
+        return NGX_OK;
+    }
+
+    if (p->length == 0) {
+
+        ngx_log_error(NGX_LOG_WARN, p->log, 0,
+                      "upstream sent more data than specified in "
+                      "\"Content-Length\" header");
+
+        r = p->input_ctx;
+        r->upstream->keepalive = 0;
+        p->upstream_done = 1;
+
+        return NGX_OK;
+    }
+
     cl = ngx_chain_get_free_buf(p->pool, &p->free);
     if (cl == NULL) {
         return NGX_ERROR;
@@ -2042,20 +2061,23 @@ ngx_http_proxy_copy_filter(ngx_event_pip
         return NGX_OK;
     }
 
+    if (b->last - b->pos > p->length) {
+
+        ngx_log_error(NGX_LOG_WARN, p->log, 0,
+                      "upstream sent more data than specified in "
+                      "\"Content-Length\" header");
+
+        b->last = b->pos + p->length;
+        p->upstream_done = 1;
+
+        return NGX_OK;
+    }
+
     p->length -= b->last - b->pos;
 
     if (p->length == 0) {
         r = p->input_ctx;
-        p->upstream_done = 1;
         r->upstream->keepalive = !r->upstream->headers_in.connection_close;
-
-    } else if (p->length < 0) {
-        r = p->input_ctx;
-        p->upstream_done = 1;
-
-        ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
-                      "upstream sent more data than specified in "
-                      "\"Content-Length\" header");
     }
 
     return NGX_OK;
@@ -2227,6 +2249,18 @@ ngx_http_proxy_non_buffered_copy_filter(
         return NGX_OK;
     }
 
+    if (bytes > u->length) {
+
+        ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
+                      "upstream sent more data than specified in "
+                      "\"Content-Length\" header");
+
+        cl->buf->last = cl->buf->pos + u->length;
+        u->length = 0;
+
+        return NGX_OK;
+    }
+
     u->length -= bytes;
 
     if (u->length == 0) {

From mdounin at mdounin.ru  Mon Jul  6 18:00:56 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 06 Jul 2020 18:00:56 +0000
Subject: [nginx] Proxy: detection of data after final chunk.
Message-ID: <hg.d225b70d38b6.1594058456.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/d225b70d38b6
branches:  
changeset: 7676:d225b70d38b6
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Mon Jul 06 18:36:20 2020 +0300
description:
Proxy: detection of data after final chunk.

Previously, additional data after final chunk was either ignored
(in the same buffer, or during unbuffered proxying) or sent to the
client (in the next buffer already if it was already read from the
socket).  Now additional data are properly detected and ignored
in all cases.  Additionally, a warning is now logged and keepalive
is disabled in the connection.

diffstat:

 src/http/modules/ngx_http_proxy_module.c |  31 ++++++++++++++++++++++++++++++-
 1 files changed, 30 insertions(+), 1 deletions(-)

diffs (57 lines):

diff -r 9afa45068b8f -r d225b70d38b6 src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c	Mon Jul 06 18:36:19 2020 +0300
+++ b/src/http/modules/ngx_http_proxy_module.c	Mon Jul 06 18:36:20 2020 +0300
@@ -2104,6 +2104,23 @@ ngx_http_proxy_chunked_filter(ngx_event_
         return NGX_ERROR;
     }
 
+    if (p->upstream_done) {
+        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, p->log, 0,
+                       "http proxy data after close");
+        return NGX_OK;
+    }
+
+    if (p->length == 0) {
+
+        ngx_log_error(NGX_LOG_WARN, p->log, 0,
+                      "upstream sent data after final chunk");
+
+        r->upstream->keepalive = 0;
+        p->upstream_done = 1;
+
+        return NGX_OK;
+    }
+
     b = NULL;
     prev = &buf->shadow;
 
@@ -2166,9 +2183,15 @@ ngx_http_proxy_chunked_filter(ngx_event_
 
             /* a whole response has been parsed successfully */
 
-            p->upstream_done = 1;
+            p->length = 0;
             r->upstream->keepalive = !r->upstream->headers_in.connection_close;
 
+            if (buf->pos != buf->last) {
+                ngx_log_error(NGX_LOG_WARN, p->log, 0,
+                              "upstream sent data after final chunk");
+                r->upstream->keepalive = 0;
+            }
+
             break;
         }
 
@@ -2347,6 +2370,12 @@ ngx_http_proxy_non_buffered_chunked_filt
             u->keepalive = !u->headers_in.connection_close;
             u->length = 0;
 
+            if (buf->pos != buf->last) {
+                ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
+                              "upstream sent data after final chunk");
+                u->keepalive = 0;
+            }
+
             break;
         }
 

From mdounin at mdounin.ru  Mon Jul  6 18:00:59 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 06 Jul 2020 18:00:59 +0000
Subject: [nginx] Proxy: style.
Message-ID: <hg.a786e491d08d.1594058459.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/a786e491d08d
branches:  
changeset: 7677:a786e491d08d
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Mon Jul 06 18:36:21 2020 +0300
description:
Proxy: style.

diffstat:

 src/http/modules/ngx_http_proxy_module.c |  4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diffs (19 lines):

diff -r d225b70d38b6 -r a786e491d08d src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c	Mon Jul 06 18:36:20 2020 +0300
+++ b/src/http/modules/ngx_http_proxy_module.c	Mon Jul 06 18:36:21 2020 +0300
@@ -2206,13 +2206,13 @@ ngx_http_proxy_chunked_filter(ngx_event_
 
         /* invalid response */
 
-        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+        ngx_log_error(NGX_LOG_ERR, p->log, 0,
                       "upstream sent invalid chunked response");
 
         return NGX_ERROR;
     }
 
-    ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+    ngx_log_debug2(NGX_LOG_DEBUG_HTTP, p->log, 0,
                    "http proxy chunked state %ui, length %O",
                    ctx->chunked.state, p->length);
 

From mdounin at mdounin.ru  Mon Jul  6 18:01:02 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 06 Jul 2020 18:01:02 +0000
Subject: [nginx] Upstream: drop extra data sent by upstream.
Message-ID: <hg.bffcc5af1d72.1594058462.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/bffcc5af1d72
branches:  
changeset: 7678:bffcc5af1d72
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Mon Jul 06 18:36:22 2020 +0300
description:
Upstream: drop extra data sent by upstream.

Previous behaviour was to pass everything to the client, but this
seems to be suboptimal and causes issues (ticket #1695).  Fix is to
drop extra data instead, as it naturally happens in most clients.

This change covers generic buffered and unbuffered filters as used
in the scgi and uwsgi modules.  Appropriate input filter init
handlers are provided by the scgi and uwsgi modules to set corresponding
lengths.

Note that for responses to HEAD requests there is an exception:
we do allow any response length.  This is because responses to HEAD
requests might be actual full responses, and it is up to nginx
to remove the response body.  If caching is enabled, only full
responses matching the Content-Length header will be cached
(see b779728b180c).

diffstat:

 src/event/ngx_event_pipe.c               |  28 ++++++++++++++++++++++++
 src/http/modules/ngx_http_scgi_module.c  |  36 ++++++++++++++++++++++++++++++++
 src/http/modules/ngx_http_uwsgi_module.c |  36 ++++++++++++++++++++++++++++++++
 src/http/ngx_http_upstream.c             |  19 ++++++++++++----
 src/http/ngx_http_upstream.h             |   2 +
 5 files changed, 116 insertions(+), 5 deletions(-)

diffs (226 lines):

diff -r a786e491d08d -r bffcc5af1d72 src/event/ngx_event_pipe.c
--- a/src/event/ngx_event_pipe.c	Mon Jul 06 18:36:21 2020 +0300
+++ b/src/event/ngx_event_pipe.c	Mon Jul 06 18:36:22 2020 +0300
@@ -960,6 +960,22 @@ ngx_event_pipe_copy_input_filter(ngx_eve
         return NGX_OK;
     }
 
+    if (p->upstream_done) {
+        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, p->log, 0,
+                       "input data after close");
+        return NGX_OK;
+    }
+
+    if (p->length == 0) {
+        p->upstream_done = 1;
+
+        ngx_log_error(NGX_LOG_WARN, p->log, 0,
+                      "upstream sent more data than specified in "
+                      "\"Content-Length\" header");
+
+        return NGX_OK;
+    }
+
     cl = ngx_chain_get_free_buf(p->pool, &p->free);
     if (cl == NULL) {
         return NGX_ERROR;
@@ -987,6 +1003,18 @@ ngx_event_pipe_copy_input_filter(ngx_eve
         return NGX_OK;
     }
 
+    if (b->last - b->pos > p->length) {
+
+        ngx_log_error(NGX_LOG_WARN, p->log, 0,
+                      "upstream sent more data than specified in "
+                      "\"Content-Length\" header");
+
+        b->last = b->pos + p->length;
+        p->upstream_done = 1;
+
+        return NGX_OK;
+    }
+
     p->length -= b->last - b->pos;
 
     return NGX_OK;
diff -r a786e491d08d -r bffcc5af1d72 src/http/modules/ngx_http_scgi_module.c
--- a/src/http/modules/ngx_http_scgi_module.c	Mon Jul 06 18:36:21 2020 +0300
+++ b/src/http/modules/ngx_http_scgi_module.c	Mon Jul 06 18:36:22 2020 +0300
@@ -49,6 +49,7 @@ static ngx_int_t ngx_http_scgi_create_re
 static ngx_int_t ngx_http_scgi_reinit_request(ngx_http_request_t *r);
 static ngx_int_t ngx_http_scgi_process_status_line(ngx_http_request_t *r);
 static ngx_int_t ngx_http_scgi_process_header(ngx_http_request_t *r);
+static ngx_int_t ngx_http_scgi_input_filter_init(void *data);
 static void ngx_http_scgi_abort_request(ngx_http_request_t *r);
 static void ngx_http_scgi_finalize_request(ngx_http_request_t *r, ngx_int_t rc);
 
@@ -534,6 +535,10 @@ ngx_http_scgi_handler(ngx_http_request_t
     u->pipe->input_filter = ngx_event_pipe_copy_input_filter;
     u->pipe->input_ctx = r;
 
+    u->input_filter_init = ngx_http_scgi_input_filter_init;
+    u->input_filter = ngx_http_upstream_non_buffered_filter;
+    u->input_filter_ctx = r;
+
     if (!scf->upstream.request_buffering
         && scf->upstream.pass_request_body
         && !r->headers_in.chunked)
@@ -1145,6 +1150,37 @@ ngx_http_scgi_process_header(ngx_http_re
 }
 
 
+static ngx_int_t
+ngx_http_scgi_input_filter_init(void *data)
+{
+    ngx_http_request_t   *r = data;
+    ngx_http_upstream_t  *u;
+
+    u = r->upstream;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "http scgi filter init s:%ui l:%O",
+                   u->headers_in.status_n, u->headers_in.content_length_n);
+
+    if (u->headers_in.status_n == NGX_HTTP_NO_CONTENT
+        || u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED)
+    {
+        u->pipe->length = 0;
+        u->length = 0;
+
+    } else if (r->method == NGX_HTTP_HEAD) {
+        u->pipe->length = -1;
+        u->length = -1;
+
+    } else {
+        u->pipe->length = u->headers_in.content_length_n;
+        u->length = u->headers_in.content_length_n;
+    }
+
+    return NGX_OK;
+}
+
+
 static void
 ngx_http_scgi_abort_request(ngx_http_request_t *r)
 {
diff -r a786e491d08d -r bffcc5af1d72 src/http/modules/ngx_http_uwsgi_module.c
--- a/src/http/modules/ngx_http_uwsgi_module.c	Mon Jul 06 18:36:21 2020 +0300
+++ b/src/http/modules/ngx_http_uwsgi_module.c	Mon Jul 06 18:36:22 2020 +0300
@@ -67,6 +67,7 @@ static ngx_int_t ngx_http_uwsgi_create_r
 static ngx_int_t ngx_http_uwsgi_reinit_request(ngx_http_request_t *r);
 static ngx_int_t ngx_http_uwsgi_process_status_line(ngx_http_request_t *r);
 static ngx_int_t ngx_http_uwsgi_process_header(ngx_http_request_t *r);
+static ngx_int_t ngx_http_uwsgi_input_filter_init(void *data);
 static void ngx_http_uwsgi_abort_request(ngx_http_request_t *r);
 static void ngx_http_uwsgi_finalize_request(ngx_http_request_t *r,
     ngx_int_t rc);
@@ -703,6 +704,10 @@ ngx_http_uwsgi_handler(ngx_http_request_
     u->pipe->input_filter = ngx_event_pipe_copy_input_filter;
     u->pipe->input_ctx = r;
 
+    u->input_filter_init = ngx_http_uwsgi_input_filter_init;
+    u->input_filter = ngx_http_upstream_non_buffered_filter;
+    u->input_filter_ctx = r;
+
     if (!uwcf->upstream.request_buffering
         && uwcf->upstream.pass_request_body
         && !r->headers_in.chunked)
@@ -1356,6 +1361,37 @@ ngx_http_uwsgi_process_header(ngx_http_r
 }
 
 
+static ngx_int_t
+ngx_http_uwsgi_input_filter_init(void *data)
+{
+    ngx_http_request_t   *r = data;
+    ngx_http_upstream_t  *u;
+
+    u = r->upstream;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "http uwsgi filter init s:%ui l:%O",
+                   u->headers_in.status_n, u->headers_in.content_length_n);
+
+    if (u->headers_in.status_n == NGX_HTTP_NO_CONTENT
+        || u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED)
+    {
+        u->pipe->length = 0;
+        u->length = 0;
+
+    } else if (r->method == NGX_HTTP_HEAD) {
+        u->pipe->length = -1;
+        u->length = -1;
+
+    } else {
+        u->pipe->length = u->headers_in.content_length_n;
+        u->length = u->headers_in.content_length_n;
+    }
+
+    return NGX_OK;
+}
+
+
 static void
 ngx_http_uwsgi_abort_request(ngx_http_request_t *r)
 {
diff -r a786e491d08d -r bffcc5af1d72 src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c	Mon Jul 06 18:36:21 2020 +0300
+++ b/src/http/ngx_http_upstream.c	Mon Jul 06 18:36:22 2020 +0300
@@ -77,9 +77,6 @@ static void
 static void
     ngx_http_upstream_process_non_buffered_request(ngx_http_request_t *r,
     ngx_uint_t do_write);
-static ngx_int_t ngx_http_upstream_non_buffered_filter_init(void *data);
-static ngx_int_t ngx_http_upstream_non_buffered_filter(void *data,
-    ssize_t bytes);
 #if (NGX_THREADS)
 static ngx_int_t ngx_http_upstream_thread_handler(ngx_thread_task_t *task,
     ngx_file_t *file);
@@ -3705,14 +3702,14 @@ ngx_http_upstream_process_non_buffered_r
 }
 
 
-static ngx_int_t
+ngx_int_t
 ngx_http_upstream_non_buffered_filter_init(void *data)
 {
     return NGX_OK;
 }
 
 
-static ngx_int_t
+ngx_int_t
 ngx_http_upstream_non_buffered_filter(void *data, ssize_t bytes)
 {
     ngx_http_request_t  *r = data;
@@ -3748,6 +3745,18 @@ ngx_http_upstream_non_buffered_filter(vo
         return NGX_OK;
     }
 
+    if (bytes > u->length) {
+
+        ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
+                      "upstream sent more data than specified in "
+                      "\"Content-Length\" header");
+
+        cl->buf->last = cl->buf->pos + u->length;
+        u->length = 0;
+
+        return NGX_OK;
+    }
+
     u->length -= bytes;
 
     return NGX_OK;
diff -r a786e491d08d -r bffcc5af1d72 src/http/ngx_http_upstream.h
--- a/src/http/ngx_http_upstream.h	Mon Jul 06 18:36:21 2020 +0300
+++ b/src/http/ngx_http_upstream.h	Mon Jul 06 18:36:22 2020 +0300
@@ -414,6 +414,8 @@ typedef struct {
 
 ngx_int_t ngx_http_upstream_create(ngx_http_request_t *r);
 void ngx_http_upstream_init(ngx_http_request_t *r);
+ngx_int_t ngx_http_upstream_non_buffered_filter_init(void *data);
+ngx_int_t ngx_http_upstream_non_buffered_filter(void *data, ssize_t bytes);
 ngx_http_upstream_srv_conf_t *ngx_http_upstream_add(ngx_conf_t *cf,
     ngx_url_t *u, ngx_uint_t flags);
 char *ngx_http_upstream_bind_set_slot(ngx_conf_t *cf, ngx_command_t *cmd,

From mdounin at mdounin.ru  Mon Jul  6 18:01:05 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 06 Jul 2020 18:01:05 +0000
Subject: [nginx] FastCGI: protection from responses with wrong length.
Message-ID: <hg.05e42236e95b.1594058465.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/05e42236e95b
branches:  
changeset: 7679:05e42236e95b
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Mon Jul 06 18:36:23 2020 +0300
description:
FastCGI: protection from responses with wrong length.

Previous behaviour was to pass everything to the client, but this
seems to be suboptimal and causes issues (ticket #1695).  Fix is to
drop extra data instead, as it naturally happens in most clients.

Additionally, we now also issue a warning if the response is too
short, and make sure the fact it is truncated is propagated to the
client.  The u->error flag is introduced to make it possible to
propagate the error to the client in case of unbuffered proxying.

For responses to HEAD requests there is an exception: we do allow
both responses without body and responses with body matching the
Content-Length header.

diffstat:

 src/http/modules/ngx_http_fastcgi_module.c |  128 ++++++++++++++++++++++++----
 src/http/ngx_http_upstream.c               |    3 +-
 src/http/ngx_http_upstream.h               |    1 +
 3 files changed, 113 insertions(+), 19 deletions(-)

diffs (236 lines):

diff -r bffcc5af1d72 -r 05e42236e95b src/http/modules/ngx_http_fastcgi_module.c
--- a/src/http/modules/ngx_http_fastcgi_module.c	Mon Jul 06 18:36:22 2020 +0300
+++ b/src/http/modules/ngx_http_fastcgi_module.c	Mon Jul 06 18:36:23 2020 +0300
@@ -81,12 +81,15 @@ typedef struct {
     size_t                         length;
     size_t                         padding;
 
+    off_t                          rest;
+
     ngx_chain_t                   *free;
     ngx_chain_t                   *busy;
 
     unsigned                       fastcgi_stdout:1;
     unsigned                       large_stderr:1;
     unsigned                       header_sent:1;
+    unsigned                       closed:1;
 
     ngx_array_t                   *split_parts;
 
@@ -2075,13 +2078,31 @@ ngx_http_fastcgi_process_header(ngx_http
 static ngx_int_t
 ngx_http_fastcgi_input_filter_init(void *data)
 {
-    ngx_http_request_t           *r = data;
+    ngx_http_request_t  *r = data;
+
+    ngx_http_upstream_t          *u;
+    ngx_http_fastcgi_ctx_t       *f;
     ngx_http_fastcgi_loc_conf_t  *flcf;
 
+    u = r->upstream;
+
+    f = ngx_http_get_module_ctx(r, ngx_http_fastcgi_module);
     flcf = ngx_http_get_module_loc_conf(r, ngx_http_fastcgi_module);
 
-    r->upstream->pipe->length = flcf->keep_conn ?
-                                (off_t) sizeof(ngx_http_fastcgi_header_t) : -1;
+    u->pipe->length = flcf->keep_conn ?
+                      (off_t) sizeof(ngx_http_fastcgi_header_t) : -1;
+
+    if (u->headers_in.status_n == NGX_HTTP_NO_CONTENT
+        || u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED)
+    {
+        f->rest = 0;
+
+    } else if (r->method == NGX_HTTP_HEAD) {
+        f->rest = -2;
+
+    } else {
+        f->rest = u->headers_in.content_length_n;
+    }
 
     return NGX_OK;
 }
@@ -2106,6 +2127,15 @@ ngx_http_fastcgi_input_filter(ngx_event_
     f = ngx_http_get_module_ctx(r, ngx_http_fastcgi_module);
     flcf = ngx_http_get_module_loc_conf(r, ngx_http_fastcgi_module);
 
+    if (p->upstream_done || f->closed) {
+        r->upstream->keepalive = 0;
+
+        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, p->log, 0,
+                       "http fastcgi data after close");
+
+        return NGX_OK;
+    }
+
     b = NULL;
     prev = &buf->shadow;
 
@@ -2128,13 +2158,25 @@ ngx_http_fastcgi_input_filter(ngx_event_
             if (f->type == NGX_HTTP_FASTCGI_STDOUT && f->length == 0) {
                 f->state = ngx_http_fastcgi_st_padding;
 
+                ngx_log_debug0(NGX_LOG_DEBUG_HTTP, p->log, 0,
+                               "http fastcgi closed stdout");
+
+                if (f->rest > 0) {
+                    ngx_log_error(NGX_LOG_ERR, p->log, 0,
+                                  "upstream prematurely closed "
+                                  "FastCGI stdout");
+
+                    p->upstream_error = 1;
+                    p->upstream_eof = 0;
+                    f->closed = 1;
+
+                    break;
+                }
+
                 if (!flcf->keep_conn) {
                     p->upstream_done = 1;
                 }
 
-                ngx_log_debug0(NGX_LOG_DEBUG_HTTP, p->log, 0,
-                               "http fastcgi closed stdout");
-
                 continue;
             }
 
@@ -2143,6 +2185,18 @@ ngx_http_fastcgi_input_filter(ngx_event_
                 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, p->log, 0,
                                "http fastcgi sent end request");
 
+                if (f->rest > 0) {
+                    ngx_log_error(NGX_LOG_ERR, p->log, 0,
+                                  "upstream prematurely closed "
+                                  "FastCGI request");
+
+                    p->upstream_error = 1;
+                    p->upstream_eof = 0;
+                    f->closed = 1;
+
+                    break;
+                }
+
                 if (!flcf->keep_conn) {
                     p->upstream_done = 1;
                     break;
@@ -2289,15 +2343,31 @@ ngx_http_fastcgi_input_filter(ngx_event_
             f->pos += f->length;
             b->last = f->pos;
 
-            continue;
+        } else {
+            f->length -= f->last - f->pos;
+            f->pos = f->last;
+            b->last = f->last;
+        }
+
+        if (f->rest == -2) {
+            f->rest = r->upstream->headers_in.content_length_n;
         }
 
-        f->length -= f->last - f->pos;
-
-        b->last = f->last;
-
-        break;
-
+        if (f->rest >= 0) {
+
+            if (b->last - b->pos > f->rest) {
+                ngx_log_error(NGX_LOG_WARN, p->log, 0,
+                              "upstream sent more data than specified in "
+                              "\"Content-Length\" header");
+
+                b->last = b->pos + f->rest;
+                p->upstream_done = 1;
+
+                break;
+            }
+
+            f->rest -= b->last - b->pos;
+        }
     }
 
     if (flcf->keep_conn) {
@@ -2391,6 +2461,14 @@ ngx_http_fastcgi_non_buffered_filter(voi
 
             if (f->type == NGX_HTTP_FASTCGI_END_REQUEST) {
 
+                if (f->rest > 0) {
+                    ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                                  "upstream prematurely closed "
+                                  "FastCGI request");
+                    u->error = 1;
+                    break;
+                }
+
                 if (f->pos + f->padding < f->last) {
                     u->length = 0;
                     break;
@@ -2510,13 +2588,27 @@ ngx_http_fastcgi_non_buffered_filter(voi
             f->pos += f->length;
             b->last = f->pos;
 
-            continue;
+        } else {
+            f->length -= f->last - f->pos;
+            f->pos = f->last;
+            b->last = f->last;
         }
 
-        f->length -= f->last - f->pos;
-        b->last = f->last;
-
-        break;
+        if (f->rest >= 0) {
+
+            if (b->last - b->pos > f->rest) {
+                ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
+                              "upstream sent more data than specified in "
+                              "\"Content-Length\" header");
+
+                b->last = b->pos + f->rest;
+                u->length = 0;
+
+                break;
+            }
+
+            f->rest -= b->last - b->pos;
+        }
     }
 
     return NGX_OK;
diff -r bffcc5af1d72 -r 05e42236e95b src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c	Mon Jul 06 18:36:22 2020 +0300
+++ b/src/http/ngx_http_upstream.c	Mon Jul 06 18:36:23 2020 +0300
@@ -1916,6 +1916,7 @@ ngx_http_upstream_reinit(ngx_http_reques
 
     u->keepalive = 0;
     u->upgrade = 0;
+    u->error = 0;
 
     ngx_memzero(&u->headers_in, sizeof(ngx_http_upstream_headers_in_t));
     u->headers_in.content_length_n = -1;
@@ -3624,7 +3625,7 @@ ngx_http_upstream_process_non_buffered_r
                     return;
                 }
 
-                if (upstream->read->error) {
+                if (upstream->read->error || u->error) {
                     ngx_http_upstream_finalize_request(r, u,
                                                        NGX_HTTP_BAD_GATEWAY);
                     return;
diff -r bffcc5af1d72 -r 05e42236e95b src/http/ngx_http_upstream.h
--- a/src/http/ngx_http_upstream.h	Mon Jul 06 18:36:22 2020 +0300
+++ b/src/http/ngx_http_upstream.h	Mon Jul 06 18:36:23 2020 +0300
@@ -391,6 +391,7 @@ struct ngx_http_upstream_s {
     unsigned                         buffering:1;
     unsigned                         keepalive:1;
     unsigned                         upgrade:1;
+    unsigned                         error:1;
 
     unsigned                         request_sent:1;
     unsigned                         request_body_sent:1;

From mdounin at mdounin.ru  Mon Jul  6 18:01:08 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 06 Jul 2020 18:01:08 +0000
Subject: [nginx] gRPC: generate error when response size is wrong.
Message-ID: <hg.39501ce97e29.1594058468.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/39501ce97e29
branches:  
changeset: 7680:39501ce97e29
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Mon Jul 06 18:36:25 2020 +0300
description:
gRPC: generate error when response size is wrong.

As long as the "Content-Length" header is given, we now make sure
it exactly matches the size of the response.  If it doesn't,
the response is considered malformed and must not be forwarded
(https://tools.ietf.org/html/rfc7540#section-8.1.2.6).  While it
is not really possible to "not forward" the response which is already
being forwarded, we generate an error instead, which is the closest
equivalent.

Previous behaviour was to pass everything to the client, but this
seems to be suboptimal and causes issues (ticket #1695).  Also this
directly contradicts HTTP/2 specification requirements.

Note that the new behaviour for the gRPC proxy is more strict than that
applied in other variants of proxying.  This is intentional, as HTTP/2
specification requires us to do so, while in other types of proxying
malformed responses from backends are well known and historically
tolerated.

diffstat:

 src/http/modules/ngx_http_grpc_module.c |  39 ++++++++++++++++++++++++++++++++-
 1 files changed, 38 insertions(+), 1 deletions(-)

diffs (73 lines):

diff -r 05e42236e95b -r 39501ce97e29 src/http/modules/ngx_http_grpc_module.c
--- a/src/http/modules/ngx_http_grpc_module.c	Mon Jul 06 18:36:23 2020 +0300
+++ b/src/http/modules/ngx_http_grpc_module.c	Mon Jul 06 18:36:25 2020 +0300
@@ -84,6 +84,8 @@ typedef struct {
     ngx_uint_t                 pings;
     ngx_uint_t                 settings;
 
+    off_t                      length;
+
     ssize_t                    send_window;
     size_t                     recv_window;
 
@@ -1953,10 +1955,28 @@ ngx_http_grpc_filter_init(void *data)
     r = ctx->request;
     u = r->upstream;
 
-    u->length = 1;
+    if (u->headers_in.status_n == NGX_HTTP_NO_CONTENT
+        || u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED
+        || r->method == NGX_HTTP_HEAD)
+    {
+        ctx->length = 0;
+
+    } else {
+        ctx->length = u->headers_in.content_length_n;
+    }
 
     if (ctx->end_stream) {
+
+        if (ctx->length > 0) {
+            ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                          "upstream prematurely closed stream");
+            return NGX_ERROR;
+        }
+
         u->length = 0;
+
+    } else {
+        u->length = 1;
     }
 
     return NGX_OK;
@@ -1999,6 +2019,12 @@ ngx_http_grpc_filter(void *data, ssize_t
 
                 if (ctx->done) {
 
+                    if (ctx->length > 0) {
+                        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                                      "upstream prematurely closed stream");
+                        return NGX_ERROR;
+                    }
+
                     /*
                      * We have finished parsing the response and the
                      * remaining control frames.  If there are unsent
@@ -2052,6 +2078,17 @@ ngx_http_grpc_filter(void *data, ssize_t
                     return NGX_ERROR;
                 }
 
+                if (ctx->length != -1) {
+                    if ((off_t) ctx->rest > ctx->length) {
+                        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                                      "upstream sent response body larger "
+                                      "than indicated content length");
+                        return NGX_ERROR;
+                    }
+
+                    ctx->length -= ctx->rest;
+                }
+
                 if (ctx->rest > ctx->recv_window) {
                     ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                                   "upstream violated stream flow control, "

From xeioex at nginx.com  Mon Jul  6 18:39:32 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Mon, 06 Jul 2020 18:39:32 +0000
Subject: [njs] Fixed TypeScript description for NjsByteString type.
Message-ID: <hg.f2e8486b3b6d.1594060772.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/f2e8486b3b6d
branches:  
changeset: 1449:f2e8486b3b6d
user:      Artem S. Povalyukhin <artem.povaluhin at gmail.com>
date:      Sat Jun 27 19:26:41 2020 +0300
description:
Fixed TypeScript description for NjsByteString type.

NjsByteString was declared as it has no relation to string primitive type.
This causes a problem when a variable of type NjsByteString is passed into
built-in functions expecting string type.

The issue was introduced in 8f3ef384f69e.

diffstat:

 src/ts/njs_core.d.ts |  10 +++++-----
 1 files changed, 5 insertions(+), 5 deletions(-)

diffs (36 lines):

diff -r 1c729f765cfb -r f2e8486b3b6d src/ts/njs_core.d.ts
--- a/src/ts/njs_core.d.ts	Thu Jul 02 14:00:16 2020 +0000
+++ b/src/ts/njs_core.d.ts	Sat Jun 27 19:26:41 2020 +0300
@@ -14,14 +14,14 @@ interface String {
      * Serializes a Unicode string with code points up to 255
      * into a byte string, otherwise, null is returned.
      */
-    toBytes(start?: number, end?: number): NjsByteString;
+    toBytes(start?: number, end?: number): NjsByteString | null;
     /**
      * Serializes a Unicode string to a byte string using UTF8 encoding.
      */
     toUTF8(start?: number, end?: number): NjsByteString;
 }
 
-interface NjsByteString extends String {
+type NjsByteString = string & {
     /**
      * Returns a new Unicode string from a byte string where each byte is replaced
      * with a corresponding Unicode code point.
@@ -31,12 +31,12 @@ interface NjsByteString extends String {
      * Converts a byte string containing a valid UTF8 string into a Unicode string,
      * otherwise null is returned.
      */
-    fromUTF8(start?: number, end?: number): string;
+    fromUTF8(start?: number, end?: number): string | null;
     /**
      * Encodes a byte string to hex, base64, or base64url.
      */
-    toString(encoding?: "hex" | "base64" | "base64url"): string;
-}
+    toString(encoding: "hex" | "base64" | "base64url"): string;
+};
 
 type NjsStringLike = string | NjsByteString;
 

From xeioex at nginx.com  Mon Jul  6 18:39:34 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Mon, 06 Jul 2020 18:39:34 +0000
Subject: [njs] HTTP: improved TypeScript API description for detached
 subrequests.
Message-ID: <hg.2c3c34706ce2.1594060774.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/2c3c34706ce2
branches:  
changeset: 1450:2c3c34706ce2
user:      Artem S. Povalyukhin <artem.povaluhin at gmail.com>
date:      Sat Jun 27 19:26:41 2020 +0300
description:
HTTP: improved TypeScript API description for detached subrequests.

diffstat:

 nginx/ts/ngx_http_js_module.d.ts |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (12 lines):

diff -r f2e8486b3b6d -r 2c3c34706ce2 nginx/ts/ngx_http_js_module.d.ts
--- a/nginx/ts/ngx_http_js_module.d.ts	Sat Jun 27 19:26:41 2020 +0300
+++ b/nginx/ts/ngx_http_js_module.d.ts	Sat Jun 27 19:26:41 2020 +0300
@@ -350,7 +350,7 @@ interface NginxHTTPRequest {
     subrequest(uri: NjsStringLike, options: NginxSubrequestOptions | string,
                callback:(reply:NginxHTTPRequest) => void): void;
     subrequest(uri: NjsStringLike, callback:(reply:NginxHTTPRequest) => void): void;
-    subrequest(uri: NjsStringLike, options: NginxSubrequestOptions): void;
+    subrequest(uri: NjsStringLike, options: NginxSubrequestOptions & { detached: true }): void;
     /**
      * Current URI in request, normalized.
      */

From xeioex at nginx.com  Mon Jul  6 18:39:36 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Mon, 06 Jul 2020 18:39:36 +0000
Subject: [njs] HTTP: fixed TypeScript description for headers.
Message-ID: <hg.5c84e6e56386.1594060776.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/5c84e6e56386
branches:  
changeset: 1451:5c84e6e56386
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Mon Jul 06 16:44:03 2020 +0000
description:
HTTP: fixed TypeScript description for headers.

The common headers were declared using only lowercase characters.
Unknown outgoing headers are declared as having the following type:
NjsStringLike | NjsStringLike[]. This causes an issue when unknown
header is used in the context requiring only NjsStringLike.

The fix is to declare common headers using canonical casing.

The issue was introduced in 8f3ef384f69e.

diffstat:

 nginx/ts/ngx_http_js_module.d.ts |  124 +++++++++++++++++++-------------------
 1 files changed, 62 insertions(+), 62 deletions(-)

diffs (141 lines):

diff -r 2c3c34706ce2 -r 5c84e6e56386 nginx/ts/ngx_http_js_module.d.ts
--- a/nginx/ts/ngx_http_js_module.d.ts	Sat Jun 27 19:26:41 2020 +0300
+++ b/nginx/ts/ngx_http_js_module.d.ts	Mon Jul 06 16:44:03 2020 +0000
@@ -6,75 +6,75 @@ interface NginxHTTPArgs {
 
 interface NginxHeadersIn {
     // common request headers
-    readonly 'accept'?: NjsByteString;
-    readonly 'accept-charset'?: NjsByteString;
-    readonly 'accept-encoding'?: NjsByteString;
-    readonly 'accept-language'?: NjsByteString;
-    readonly 'authorization'?: NjsByteString;
-    readonly 'cache-control'?: NjsByteString;
-    readonly 'connection'?: NjsByteString;
-    readonly 'content-length'?: NjsByteString;
-    readonly 'content-type'?: NjsByteString;
-    readonly 'cookie'?: NjsByteString;
-    readonly 'date'?: NjsByteString;
-    readonly 'expect'?: NjsByteString;
-    readonly 'forwarded'?: NjsByteString;
-    readonly 'from'?: NjsByteString;
-    readonly 'host'?: NjsByteString;
-    readonly 'if-match'?: NjsByteString;
-    readonly 'if-modified-since'?: NjsByteString;
-    readonly 'if-none-match'?: NjsByteString;
-    readonly 'if-range'?: NjsByteString;
-    readonly 'if-unmodified-since'?: NjsByteString;
-    readonly 'max-forwards'?: NjsByteString;
-    readonly 'origin'?: NjsByteString;
-    readonly 'pragma'?: NjsByteString;
-    readonly 'proxy-authorization'?: NjsByteString;
-    readonly 'range'?: NjsByteString;
-    readonly 'referer'?: NjsByteString;
-    readonly 'te'?: NjsByteString;
-    readonly 'user-agent'?: NjsByteString;
-    readonly 'upgrade'?: NjsByteString;
-    readonly 'via'?: NjsByteString;
-    readonly 'warning'?: NjsByteString;
-    readonly 'x-forwarded-for'?: NjsByteString;
+    readonly 'Accept'?: NjsByteString;
+    readonly 'Accept-Charset'?: NjsByteString;
+    readonly 'Accept-Encoding'?: NjsByteString;
+    readonly 'Accept-Language'?: NjsByteString;
+    readonly 'Authorization'?: NjsByteString;
+    readonly 'Cache-Control'?: NjsByteString;
+    readonly 'Connection'?: NjsByteString;
+    readonly 'Content-Length'?: NjsByteString;
+    readonly 'Content-Type'?: NjsByteString;
+    readonly 'Cookie'?: NjsByteString;
+    readonly 'Date'?: NjsByteString;
+    readonly 'Expect'?: NjsByteString;
+    readonly 'Forwarded'?: NjsByteString;
+    readonly 'From'?: NjsByteString;
+    readonly 'Host'?: NjsByteString;
+    readonly 'If-Match'?: NjsByteString;
+    readonly 'If-Modified-Since'?: NjsByteString;
+    readonly 'If-None-Match'?: NjsByteString;
+    readonly 'If-Range'?: NjsByteString;
+    readonly 'If-Unmodified-Since'?: NjsByteString;
+    readonly 'Max-Forwards'?: NjsByteString;
+    readonly 'Origin'?: NjsByteString;
+    readonly 'Pragma'?: NjsByteString;
+    readonly 'Proxy-Authorization'?: NjsByteString;
+    readonly 'Range'?: NjsByteString;
+    readonly 'Referer'?: NjsByteString;
+    readonly 'TE'?: NjsByteString;
+    readonly 'User-Agent'?: NjsByteString;
+    readonly 'Upgrade'?: NjsByteString;
+    readonly 'Via'?: NjsByteString;
+    readonly 'Warning'?: NjsByteString;
+    readonly 'X-Forwarded-For'?: NjsByteString;
 
     readonly [prop: string]: NjsByteString;
 }
 
 interface NginxHeadersOut {
     // common response headers
-    'age'?: NjsStringLike;
-    'allow'?: NjsStringLike;
-    'alt-svc'?: NjsStringLike;
-    'cache-control'?: NjsStringLike;
-    'connection'?: NjsStringLike;
-    'content-disposition'?: NjsStringLike;
-    'content-encoding'?: NjsStringLike;
-    'content-language'?: NjsStringLike;
-    'content-length'?: NjsStringLike;
-    'content-location'?: NjsStringLike;
-    'content-range'?: NjsStringLike;
-    'content-type'?: NjsStringLike;
-    'date'?: NjsStringLike;
-    'etag'?: NjsStringLike;
-    'expires'?: NjsStringLike;
-    'last-modified'?: NjsStringLike;
-    'link'?: NjsStringLike;
-    'location'?: NjsStringLike;
-    'pragma'?: NjsStringLike;
-    'proxy-authenticate'?: NjsStringLike;
-    'retry-after'?: NjsStringLike;
-    'server'?: NjsStringLike;
-    'trailer'?: NjsStringLike;
-    'transfer-encoding'?: NjsStringLike;
-    'upgrade'?: NjsStringLike;
-    'vary'?: NjsStringLike;
-    'via'?: NjsStringLike;
-    'warning'?: NjsStringLike;
-    'www-authenticate'?: NjsStringLike;
+    'Age'?: NjsStringLike;
+    'Allow'?: NjsStringLike;
+    'Alt-Svc'?: NjsStringLike;
+    'Cache-Control'?: NjsStringLike;
+    'Connection'?: NjsStringLike;
+    'Content-Disposition'?: NjsStringLike;
+    'Content-Encoding'?: NjsStringLike;
+    'Content-Language'?: NjsStringLike;
+    'Content-Length'?: NjsStringLike;
+    'Content-Location'?: NjsStringLike;
+    'Content-Range'?: NjsStringLike;
+    'Content-Type'?: NjsStringLike;
+    'Date'?: NjsStringLike;
+    'ETag'?: NjsStringLike;
+    'Expires'?: NjsStringLike;
+    'Last-Modified'?: NjsStringLike;
+    'Link'?: NjsStringLike;
+    'Location'?: NjsStringLike;
+    'Pragma'?: NjsStringLike;
+    'Proxy-Authenticate'?: NjsStringLike;
+    'Retry-After'?: NjsStringLike;
+    'Server'?: NjsStringLike;
+    'Trailer'?: NjsStringLike;
+    'Transfer-Encoding'?: NjsStringLike;
+    'Upgrade'?: NjsStringLike;
+    'Vary'?: NjsStringLike;
+    'Via'?: NjsStringLike;
+    'Warning'?: NjsStringLike;
+    'WWW-Authenticate'?: NjsStringLike;
 
-    'set-cookie'?: NjsStringLike[];
+    'Set-Cookie'?: NjsStringLike[];
 
     [prop: string]: NjsStringLike | NjsStringLike[];
 }

From xeioex at nginx.com  Mon Jul  6 18:39:38 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Mon, 06 Jul 2020 18:39:38 +0000
Subject: [njs] Added TypeScript test.
Message-ID: <hg.29ff89055e29.1594060778.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/29ff89055e29
branches:  
changeset: 1452:29ff89055e29
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Mon Jul 06 18:37:13 2020 +0000
description:
Added TypeScript test.

diffstat:

 auto/make       |   3 ++
 test/ts/test.ts |  66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 0 deletions(-)

diffs (83 lines):

diff -r 5c84e6e56386 -r 29ff89055e29 auto/make
--- a/auto/make	Mon Jul 06 16:44:03 2020 +0000
+++ b/auto/make	Mon Jul 06 18:37:13 2020 +0000
@@ -265,6 +265,9 @@ ts:
 	cp nginx/ts/*.ts $NJS_BUILD_DIR/ts/
 	cp src/ts/*.ts $NJS_BUILD_DIR/ts/
 
+ts_test: ts
+	tsc ./test/ts/test.ts
+
 dist:
 	NJS_VER=`grep NJS_VERSION src/njs.h | sed -e 's/.*"\(.*\)".*/\1/'`; \\
 	rm -rf njs-\$\${NJS_VER} \\
diff -r 5c84e6e56386 -r 29ff89055e29 test/ts/test.ts
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/ts/test.ts	Mon Jul 06 18:37:13 2020 +0000
@@ -0,0 +1,66 @@
+/// <reference path="../../build/ts/ngx_http_js_module.d.ts" />
+
+function handler(r: NginxHTTPRequest) {
+    var bs: NjsByteString;
+    var s: string;
+
+    // builtin string vs NjsByteString
+
+    s = 'ordinary string';
+    bs = String.bytesFrom('000000', 'hex');
+    bs = s.toBytes();
+    bs = s.toUTF8();
+    bs.fromBytes(null, null);
+
+    s = bs + '';
+
+    // r.uri
+
+    if (r.uri == '/') {
+    }
+
+    // r.args
+
+    bs = r.args.x;
+    bs = r.args[1];
+    s = r.args.x.fromUTF8();
+    s = r.args.x + '';
+
+    // r.headersIn
+
+    r.headersIn['Accept'].fromBytes() == 'dddd';
+
+    // r.headersOut
+
+    r.headersOut['Content-Type'] = 'text/plain';
+    // Warning: r.headersOut['Content-Type'] = ['a', 'b'];
+    r.headersOut['Connection'] = undefined;
+    r.headersOut['Connection'] = null;
+
+    r.headersOut['Set-Cookie'] = ['aaa', 'bbb'];
+    r.headersOut['Foo'] = ['aaa', 'bbb'];
+
+    r.subrequest('/uri', reply => r.return(200, reply.headersOut["Location"]));
+
+    // r.log
+
+    r.log(bs);
+    r.log(r.headersOut['Connection']);
+
+    // r.variables
+
+    r.variables.a == 'a';
+    r.variables.cookie_a = 'b';
+
+    // r.subrequest
+    r.subrequest('/p/sub1').then(reply => r.return(reply.status));
+    r.subrequest('/p/sub2', reply => r.return(reply.status));
+    r.subrequest('/p/sub3', {detached:true});
+    r.subrequest('/p/sub4', 'a=1&b=2').then(reply => r.return(reply.status,
+                                        JSON.stringify(JSON.parse(reply.responseBody))));
+
+    // builtin objects
+
+    njs.dump('asdf');
+    njs.version != process.argv[1];
+}

From xeioex at nginx.com  Tue Jul  7 12:25:25 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 07 Jul 2020 12:25:25 +0000
Subject: [njs] Fixed potential NULL-pointer dereference (CID 1464584).
Message-ID: <hg.7a8e3f0d52a6.1594124725.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/7a8e3f0d52a6
branches:  
changeset: 1453:7a8e3f0d52a6
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 07 12:23:42 2020 +0000
description:
Fixed potential NULL-pointer dereference (CID 1464584).

diffstat:

 src/njs_generator.c |  9 +++++++--
 1 files changed, 7 insertions(+), 2 deletions(-)

diffs (19 lines):

diff -r 29ff89055e29 -r 7a8e3f0d52a6 src/njs_generator.c
--- a/src/njs_generator.c	Mon Jul 06 18:37:13 2020 +0000
+++ b/src/njs_generator.c	Tue Jul 07 12:23:42 2020 +0000
@@ -571,8 +571,13 @@ njs_lookup_line(njs_vm_code_t *code, uin
     njs_uint_t         n;
     njs_vm_line_num_t  *map;
 
-    n = (code->lines != NULL) ? code->lines->items : 0;
-    map = (njs_vm_line_num_t *) code->lines->start;
+    n = 0;
+    map = NULL;
+
+    if (code->lines != NULL) {
+        n = code->lines->items;
+        map = (njs_vm_line_num_t *) code->lines->start;
+    }
 
     while (n != 0) {
         if (offset >= map->offset && (n == 1 || offset < map[1].offset)) {

From xeioex at nginx.com  Tue Jul  7 12:25:26 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 07 Jul 2020 12:25:26 +0000
Subject: [njs] Silenced Coverity false-positive NULL-pointer dereference
 warning.
Message-ID: <hg.ceb8f0dcf48b.1594124726.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/ceb8f0dcf48b
branches:  
changeset: 1454:ceb8f0dcf48b
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 07 12:23:44 2020 +0000
description:
Silenced Coverity false-positive NULL-pointer dereference warning.

Found by Coverity (CID 1463869).

diffstat:

 src/njs_parser.c |  3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diffs (13 lines):

diff -r 7a8e3f0d52a6 -r ceb8f0dcf48b src/njs_parser.c
--- a/src/njs_parser.c	Tue Jul 07 12:23:42 2020 +0000
+++ b/src/njs_parser.c	Tue Jul 07 12:23:44 2020 +0000
@@ -1799,6 +1799,9 @@ njs_parser_property_name(njs_parser_t *p
 
     if (consume > 1) {
         token = njs_lexer_token(parser->lexer, 0);
+        if (token == NULL) {
+            return NJS_ERROR;
+        }
 
         property = njs_parser_property_name_node(parser, token);
         if (property == NULL) {

From xeioex at nginx.com  Tue Jul  7 12:25:28 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 07 Jul 2020 12:25:28 +0000
Subject: [njs] Fixed GetSubstitution() with absent namedCaptures.
Message-ID: <hg.7325a6df5036.1594124728.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/7325a6df5036
branches:  
changeset: 1455:7325a6df5036
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 07 12:23:45 2020 +0000
description:
Fixed GetSubstitution() with absent namedCaptures.

This issue was introduced in 1c729f765cfb.
Found by Clang static analyzer.

diffstat:

 src/njs_string.c         |   8 ++------
 src/test/njs_unit_test.c |  28 +++++++++++++++++++++++++++-
 2 files changed, 29 insertions(+), 7 deletions(-)

diffs (89 lines):

diff -r ceb8f0dcf48b -r 7325a6df5036 src/njs_string.c
--- a/src/njs_string.c	Tue Jul 07 12:23:44 2020 +0000
+++ b/src/njs_string.c	Tue Jul 07 12:23:45 2020 +0000
@@ -3414,7 +3414,7 @@ njs_string_get_substitution(njs_vm_t *vm
 
         case '<':
             r = njs_strlchr(p, end, '>');
-            if (r == NULL) {
+            if (groups == NULL || njs_is_undefined(groups) || r == NULL) {
                 njs_chb_append(&chain, p, 2);
                 p += 2;
                 break;
@@ -3422,10 +3422,6 @@ njs_string_get_substitution(njs_vm_t *vm
 
             p += 2;
 
-            if (groups == NULL) {
-                break;
-            }
-
             ret = njs_vm_value_string_set(vm, &name, p, r - p);
             if (njs_slow_path(ret != NJS_OK)) {
                 goto exception;
@@ -3512,7 +3508,7 @@ exception:
 
     njs_chb_destroy(&chain);
 
-    return NJS_OK;
+    return ret;
 }
 
 
diff -r ceb8f0dcf48b -r 7325a6df5036 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Tue Jul 07 12:23:44 2020 +0000
+++ b/src/test/njs_unit_test.c	Tue Jul 07 12:23:45 2020 +0000
@@ -7502,6 +7502,12 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("'abcdbe'.replace('b', '|$`X$\\'|')"),
       njs_str("a|aXcdbe|cdbe") },
 
+    { njs_str("'ABC'.replace('B', '$<g>')"),
+      njs_str("A$<g>C") },
+
+    { njs_str("'ABC'.replace('B', '$23')"),
+      njs_str("A$23C") },
+
     { njs_str("'undefined'.replace(void 0, 'x')"),
       njs_str("x") },
 
@@ -7669,6 +7675,9 @@ static njs_unit_test_t  njs_test[] =
               "uri.replace(/^\\/u\\/v1\\/[^/]*\\/([^\?]*)\\?.*(mt=[^&]*).*$/, '$1|$2')"),
       njs_str("bB|mt=42") },
 
+    { njs_str("'ABC'.replace(/B/, '$<g>')"),
+      njs_str("A$<g>C") },
+
     { njs_str("'ABC'.replace(/(?<b>B)/, '|$<b>|@$<a>@')"),
       njs_str("A|B|@@C") },
 
@@ -7693,12 +7702,29 @@ static njs_unit_test_t  njs_test[] =
       njs_str("A|+|C") },
 
     { njs_str("var O = RegExp.prototype.exec;"
-              "function mangled(s) { var r = O.call(this, s); Object.defineProperty(r, '0', {enumerable:false}); "
+              "function mangled(s) { var r = O.call(this, s);"
+              "                      Object.defineProperty(r, '0', {enumerable:false}); "
               "                      return r; };"
               "RegExp.prototype.exec = mangled;"
               "'ABC'.replace(/(B)/, (m, p1, off, s) => `@${m}|${p1}|${off}|${s}@`)"),
       njs_str("A at B|B|1|ABC at C") },
 
+    { njs_str("var O = RegExp.prototype.exec;"
+              "function mangled(s) { var r = O.call(this, s);"
+              "                      Object.defineProperty(r, 'groups', {value: {g:1}}); "
+              "                      return r; };"
+              "RegExp.prototype.exec = mangled;"
+              "'ABC'.replace(/(B)/, '$<g>')"),
+      njs_str("A1C") },
+
+    { njs_str("var O = RegExp.prototype.exec;"
+              "function mangled(s) { var r = O.call(this, s);"
+              "                      Object.defineProperty(r, 'groups', {value: {get g() {throw 'OOps'}}}); "
+              "                      return r; };"
+              "RegExp.prototype.exec = mangled;"
+              "'ABC'.replace(/(B)/, '$<g>')"),
+      njs_str("OOps") },
+
     { njs_str("RegExp.prototype[Symbol.replace].call()"),
       njs_str("TypeError: \"this\" is not object") },
 

From xeioex at nginx.com  Tue Jul  7 12:25:30 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 07 Jul 2020 12:25:30 +0000
Subject: [njs] Fixed String.prototype.replace() when replacer throws exception.
Message-ID: <hg.e88aef77c653.1594124730.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/e88aef77c653
branches:  
changeset: 1456:e88aef77c653
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 07 12:23:46 2020 +0000
description:
Fixed String.prototype.replace() when replacer throws exception.

This issue was introduced in 1c729f765cfb.
Found by Clang static analyzer.

diffstat:

 src/njs_string.c         |  4 ++++
 src/test/njs_unit_test.c |  3 +++
 2 files changed, 7 insertions(+), 0 deletions(-)

diffs (27 lines):

diff -r 7325a6df5036 -r e88aef77c653 src/njs_string.c
--- a/src/njs_string.c	Tue Jul 07 12:23:45 2020 +0000
+++ b/src/njs_string.c	Tue Jul 07 12:23:46 2020 +0000
@@ -3601,6 +3601,10 @@ njs_string_prototype_replace(njs_vm_t *v
                                 njs_value_arg(&njs_value_undefined),
                                 arguments, 3, &retval);
 
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
         ret = njs_value_to_string(vm, &retval, &retval);
         if (njs_slow_path(ret != NJS_OK)) {
             return NJS_ERROR;
diff -r 7325a6df5036 -r e88aef77c653 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Tue Jul 07 12:23:45 2020 +0000
+++ b/src/test/njs_unit_test.c	Tue Jul 07 12:23:46 2020 +0000
@@ -7523,6 +7523,9 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("'12345'.replace(3, () => ({toString: () => 'aaaa'}))"),
       njs_str("12aaaa45") },
 
+    { njs_str("'ABC'.replace('B', () => {throw 'OOps'})"),
+      njs_str("OOps") },
+
     { njs_str("'abc'.replace(/a/, 'X')"),
       njs_str("Xbc") },
 

From xeioex at nginx.com  Tue Jul  7 12:25:32 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 07 Jul 2020 12:25:32 +0000
Subject: [njs] Removed dead store assignment after 1c729f765cfb.
Message-ID: <hg.63b43891bc51.1594124732.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/63b43891bc51
branches:  
changeset: 1457:63b43891bc51
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 07 12:23:47 2020 +0000
description:
Removed dead store assignment after 1c729f765cfb.

Found by Clang static analyzer.

diffstat:

 src/njs_string.c |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (12 lines):

diff -r e88aef77c653 -r 63b43891bc51 src/njs_string.c
--- a/src/njs_string.c	Tue Jul 07 12:23:46 2020 +0000
+++ b/src/njs_string.c	Tue Jul 07 12:23:47 2020 +0000
@@ -3631,7 +3631,7 @@ njs_string_prototype_replace(njs_vm_t *v
 
     r = njs_cpymem(r, string.start, p - string.start);
     r = njs_cpymem(r, ret_string.start, ret_string.size);
-    r = njs_cpymem(r, p + s.size, string.size - s.size - (p - string.start));
+    memcpy(r, p + s.size, string.size - s.size - (p - string.start));
 
     return NJS_OK;
 }

From xeioex at nginx.com  Tue Jul  7 12:25:34 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 07 Jul 2020 12:25:34 +0000
Subject: [njs] Version 0.4.2.
Message-ID: <hg.b409e86fd02a.1594124734.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/b409e86fd02a
branches:  
changeset: 1458:b409e86fd02a
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 07 12:23:48 2020 +0000
description:
Version 0.4.2.

diffstat:

 CHANGES |  39 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 39 insertions(+), 0 deletions(-)

diffs (46 lines):

diff -r 63b43891bc51 -r b409e86fd02a CHANGES
--- a/CHANGES	Tue Jul 07 12:23:47 2020 +0000
+++ b/CHANGES	Tue Jul 07 12:23:48 2020 +0000
@@ -1,3 +1,42 @@
+
+Changes with njs 0.4.2                                           07 Jul 2020
+
+    Core:
+
+    *) Feature: added RegExp.prototype[Symbol.replace].
+
+    *) Feature: introduced line level backtrace.
+
+    *) Feature: added %TypedArray%.prototype.sort().
+
+    *) Feature: extended "fs" module. Added mkdir(), readdir(), rmdir()
+       and friends.
+       Thanks to Artem S. Povalyukhin.
+
+    *) Improvement: parser refactoring.
+
+    *) Bugfix: fixed TypedScript API description for HTTP headers.
+
+    *) Bugfix: fixed TypedScript API description for NjsByteString type.
+
+    *) Bugfix: fixed String.prototype.repeat() according to the
+       specification.
+
+    *) Bugfix: fixed parsing of flags for regexp literals.
+
+    *) Bugfix: fixed index generation for global objects in generator.
+
+    *) Bugfix: fixed String.prototype.replace() according to the
+       specification.
+
+    *) Bugfix: fixed %TypedArray%.prototype.copyWithin() with nonzero
+       byte offset.
+
+    *) Bugfix: fixed Array.prototype.splice() for sparse arrays.
+
+    *) Bugfix: fixed Array.prototype.reverse() for sparse arrays.
+
+    *) Bugfix: fixed Array.prototype.sort() for sparse arrays.
 
 Changes with njs 0.4.1                                           19 May 2020
 

From xeioex at nginx.com  Tue Jul  7 12:25:36 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 07 Jul 2020 12:25:36 +0000
Subject: [njs] Added tag 0.4.2 for changeset b409e86fd02a
Message-ID: <hg.a3dc3572efa0.1594124736.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/a3dc3572efa0
branches:  
changeset: 1459:a3dc3572efa0
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 07 12:25:03 2020 +0000
description:
Added tag 0.4.2 for changeset b409e86fd02a

diffstat:

 .hgtags |  1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diffs (8 lines):

diff -r b409e86fd02a -r a3dc3572efa0 .hgtags
--- a/.hgtags	Tue Jul 07 12:23:48 2020 +0000
+++ b/.hgtags	Tue Jul 07 12:25:03 2020 +0000
@@ -35,3 +35,4 @@ 1abb97e9d9dc07dcff2616d4c75132e6d189a6aa
 fc4eeaaf0dfe7dc7d41232f1b643bd364f1efe82 0.3.9
 6144aafa1472fbdf79bc9d1f858555938ee08452 0.4.0
 9400790bf53843001f94c77b47bc99b05518af78 0.4.1
+b409e86fd02a6f2cb3d741a41b6562471e1b66ef 0.4.2

From weijunjie at didiglobal.com  Tue Jul  7 14:45:00 2020
From: weijunjie at didiglobal.com (=?gb2312?B?zrq/ob3cIEp1bmppZSBXZWk=?=)
Date: Tue, 7 Jul 2020 14:45:00 +0000
Subject: Thrift proxy code contribution
In-Reply-To: <1594132884832.70253@didiglobal.com>
References: <1594132884832.70253@didiglobal.com>
Message-ID: <1594133067108.54868@didiglobal.com>

Hi Maxim Dounin, I'm glad to receive your reply.

The nginx directories are as follow:

auto
conf
contrib
docs
misc
objs
src
|-core
|-event
|-http
|-mail
|-misc
|-os
|-stream
|-thrift
  |-modules



Just like supporting tcp proxy, we developed a series of modules to support thrift protocol proxying.

No matter using or compiling, these modules are optional.


Config example:

worker_rlimit_nofile 204800;
worker_processes 1;?
error_log  logs/error.log error;
pid        run/nginx.pid;
http{
    ...
}
thrift {
    access_log logs/access.log;

    server {
        listen 8000;
        server_name counter; # counter is service name in thrift request
        location ping {      # ping is method name in thrift request
            proxy_pass ua;
 ?       }
    }

    upstream ua {
        server 127.0.0.1:8003;
    }
}?


Most of our codes are placed in thrift directory, which consists of 1 core module:

  1.  ngx_thrift_core_module

and other functional modules:

  1.  ngx_thrift_upstream_module

  2.  ngx_thrift_proxy_module
  3.  ngx_thrift_log_module
  4.  ngx_thrift_upstream_check_module?tcp active health check
  5.  ngx_thrift_upstream_keepalive_module
  6.  ngx_http_dynamic_upstream_module : using http request to change nginx upstream endpoint without reload nginx conf
  7.  ngx_thrift_limit_req_module
  8.  ngx_thrift_dynamic_req_module: dispatching request to different upstream accoding to request content

The only things we change in original nginx are:

  1.  build script in auto directory to support compile thrift proxy core framework code into nginx
  2.  macro constance like "#define NGX_LOG_DEBUG_THRIFT 0x800" in ngx_log.h, which is non-logical

Otherwise, we developed a standalone thrift procotol encoder/decoder and we preliminarily plan to publish it independently, which will be imported in nginx as a third party lib.

Do you think out thoughts are feasible? We are really looking forward for suggestions and opinions.

By the way, must out codes support other OS like Windows or other event mechanics like poll, select, kqueue?


Thanks!

Junjie Wei

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200707/787c0095/attachment-0001.htm>

From mdounin at mdounin.ru  Tue Jul  7 15:57:20 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 07 Jul 2020 15:57:20 +0000
Subject: [nginx] nginx-1.19.1-RELEASE
Message-ID: <hg.062920e2f3bf.1594137440.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/062920e2f3bf
branches:  
changeset: 7681:062920e2f3bf
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Tue Jul 07 18:56:05 2020 +0300
description:
nginx-1.19.1-RELEASE

diffstat:

 docs/xml/nginx/changes.xml |  124 +++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 124 insertions(+), 0 deletions(-)

diffs (134 lines):

diff -r 39501ce97e29 -r 062920e2f3bf docs/xml/nginx/changes.xml
--- a/docs/xml/nginx/changes.xml	Mon Jul 06 18:36:25 2020 +0300
+++ b/docs/xml/nginx/changes.xml	Tue Jul 07 18:56:05 2020 +0300
@@ -5,6 +5,130 @@
 <change_log title="nginx">
 
 
+<changes ver="1.19.1" date="2020-07-07">
+
+<change type="change">
+<para lang="ru">
+????????? lingering_close, lingering_time ? lingering_timeout
+?????? ???????? ??? ????????????? HTTP/2.
+</para>
+<para lang="en">
+the "lingering_close", "lingering_time", and "lingering_timeout" directives
+now work when using HTTP/2.
+</para>
+</change>
+
+<change type="change">
+<para lang="ru">
+?????? ?????? ??????, ?????????? ????????, ?????? ?????????????.
+</para>
+<para lang="en">
+now extra data sent by a backend are always discarded.
+</para>
+</change>
+
+<change type="change">
+<para lang="ru">
+?????? ??? ????????? ??????? ????????? ?????? ?? FastCGI-???????
+nginx ???????? ????????? ??????? ????????? ????? ??????,
+????? ???? ????????? ?????????? ? ????????.
+</para>
+<para lang="en">
+now after receiving a too short response from a FastCGI server
+nginx tries to send the available part of the response to the client,
+and then closes the client connection.
+</para>
+</change>
+
+<change type="change">
+<para lang="ru">
+?????? ??? ????????? ?????? ???????????? ????? ?? gRPC-???????
+nginx ?????????? ????????? ?????? ? ???????.
+</para>
+<para lang="en">
+now after receiving a response with incorrect length from a gRPC backend
+nginx stops response processing with an error.
+</para>
+</change>
+
+<change type="feature">
+<para lang="ru">
+???????? min_free ? ?????????? proxy_cache_path, fastcgi_cache_path,
+scgi_cache_path ? uwsgi_cache_path.<br/>
+??????? Adam Bambuch.
+</para>
+<para lang="en">
+the "min_free" parameter of the "proxy_cache_path", "fastcgi_cache_path",
+"scgi_cache_path", and "uwsgi_cache_path" directives.<br/>
+Thanks to Adam Bambuch.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+nginx ?? ?????? unix domain listen-??????
+??? ??????? ?????????? ?? ??????? SIGQUIT.
+</para>
+<para lang="en">
+nginx did not delete unix domain listen sockets
+during graceful shutdown on the SIGQUIT signal.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+UDP-?????? ???????? ??????? ?? ??????????????.
+</para>
+<para lang="en">
+zero length UDP datagrams were not proxied.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+????????????? ?? uwsgi-??????? ? ?????????????? SSL ????? ?? ????????.<br/>
+??????? Guanzhong Chen.
+</para>
+<para lang="en">
+proxying to uwsgi backends using SSL might not work.<br/>
+Thanks to Guanzhong Chen.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+? ????????? ?????? ??? ????????????? ????????? ssl_ocsp.
+</para>
+<para lang="en">
+in error handling when using the "ssl_ocsp" directive.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+??? ????????????? ???????? ?????? XFS ? NFS
+?????? ???? ?? ????? ??? ????????? ???????????.
+</para>
+<para lang="en">
+on XFS and NFS file systems
+disk cache size might be calculated incorrectly.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+???? ?????? memcached ????????? ???????????? ?????,
+? ????? ????? ?????????? ????????? "negative size buf in writer".
+</para>
+<para lang="en">
+"negative size buf in writer" alerts might appear in logs
+if a memcached server returned a malformed response.
+</para>
+</change>
+
+</changes>
+
+
 <changes ver="1.19.0" date="2020-05-26">
 
 <change type="feature">

From mdounin at mdounin.ru  Tue Jul  7 15:57:23 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 07 Jul 2020 15:57:23 +0000
Subject: [nginx] release-1.19.1 tag
Message-ID: <hg.2ab0ecfe5a5f.1594137443.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/2ab0ecfe5a5f
branches:  
changeset: 7682:2ab0ecfe5a5f
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Tue Jul 07 18:56:06 2020 +0300
description:
release-1.19.1 tag

diffstat:

 .hgtags |  1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diffs (8 lines):

diff -r 062920e2f3bf -r 2ab0ecfe5a5f .hgtags
--- a/.hgtags	Tue Jul 07 18:56:05 2020 +0300
+++ b/.hgtags	Tue Jul 07 18:56:06 2020 +0300
@@ -450,3 +450,4 @@ fdacd273711ddf20f778c1fb91529ab53979a454
 5e8d52bca714d4b85284ddb649d1ba4a3ca978a8 release-1.17.9
 c44970de01474f6f3e01b0adea85ec1d03e3a5f2 release-1.17.10
 cbe6ba650211541310618849168631ce0b788f35 release-1.19.0
+062920e2f3bf871ef7a3d8496edec1b3065faf80 release-1.19.1

From agentzh at gmail.com  Tue Jul  7 19:26:39 2020
From: agentzh at gmail.com (Yichun Zhang (agentzh))
Date: Tue, 7 Jul 2020 12:26:39 -0700
Subject: [ANN] Test::Nginx 0.27 released
Message-ID: <CAB4Tn6MivFyy1B72=x66TPxYduF05KNpsHas7qoSRv-OvGBtWg@mail.gmail.com>

Hi there,

I am happy to announce the new 0.27 release of Test::Nginx:

https://openresty.org/en/ann-test-nginx-027.html

This Perl module provides a test scaffold for automated testing in
Nginx C module or OpenResty-based Lua library development and
regression testing.

This class inherits from Test::Base, thus bringing all its declarative
power to the Nginx C module testing practices.

All of our OpenResty projects are using this test scaffold for
automated regression testing.

Enjoy!

Best regards,
Yichun
---
Yichun Zhang is the creator of OpenResty, the founder and CEO of OpenResty Inc.

From xeioex at nginx.com  Wed Jul  8 13:08:43 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Wed, 08 Jul 2020 13:08:43 +0000
Subject: [njs] Version bump.
Message-ID: <hg.69dac13b47b6.1594213723.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/69dac13b47b6
branches:  
changeset: 1460:69dac13b47b6
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Wed Jul 08 13:07:03 2020 +0000
description:
Version bump.

diffstat:

 src/njs.h |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (12 lines):

diff -r a3dc3572efa0 -r 69dac13b47b6 src/njs.h
--- a/src/njs.h	Tue Jul 07 12:25:03 2020 +0000
+++ b/src/njs.h	Wed Jul 08 13:07:03 2020 +0000
@@ -11,7 +11,7 @@
 
 #include <njs_auto_config.h>
 
-#define NJS_VERSION                 "0.4.2"
+#define NJS_VERSION                 "0.4.3"
 
 
 #include <unistd.h>                 /* STDOUT_FILENO, STDERR_FILENO */

From xeioex at nginx.com  Wed Jul  8 13:08:45 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Wed, 08 Jul 2020 13:08:45 +0000
Subject: [njs] Fixed Array.prototype.join() with TypeArray instance.
Message-ID: <hg.960402554cc9.1594213725.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/960402554cc9
branches:  
changeset: 1461:960402554cc9
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Wed Jul 08 13:07:05 2020 +0000
description:
Fixed Array.prototype.join() with TypeArray instance.

Found by Clang static analyzer.
The issue was introduced in ccfa84cea2b3.

diffstat:

 src/njs_array.c          |   3 ++-
 src/test/njs_unit_test.c |  14 +++++++++++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diffs (40 lines):

diff -r 69dac13b47b6 -r 960402554cc9 src/njs_array.c
--- a/src/njs_array.c	Wed Jul 08 13:07:03 2020 +0000
+++ b/src/njs_array.c	Wed Jul 08 13:07:05 2020 +0000
@@ -1609,7 +1609,8 @@ njs_array_prototype_join(njs_vm_t *vm, n
     njs_chb_init(&chain, vm->mem_pool);
 
     for (i = 0; i < len; i++) {
-        if (njs_fast_path(njs_object(this)->fast_array
+        if (njs_fast_path(array != NULL
+                          && array->object.fast_array
                           && njs_is_valid(&array->start[i])))
         {
             value = &array->start[i];
diff -r 69dac13b47b6 -r 960402554cc9 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Wed Jul 08 13:07:03 2020 +0000
+++ b/src/test/njs_unit_test.c	Wed Jul 08 13:07:05 2020 +0000
@@ -3973,12 +3973,20 @@ static njs_unit_test_t  njs_test[] =
       njs_str(",,,false,true,0,1") },
 
     { njs_str("var o = { toString: function() { return null } };"
-                 "[o].join()"),
+              "[o].join()"),
       njs_str("null") },
 
     { njs_str("var o = { toString: function() { return undefined } };"
-                 "[o].join()"),
-      njs_str("undefined") },
+              "[o].join()"),
+      njs_str("undefined") },
+
+    { njs_str("var a = [0,,2,3];"
+              "Object.defineProperty(Array.prototype, 1, {get: ()=> {a[32] = 32; return 1}, configurable:true});"
+              "a.join()"),
+    njs_str("0,1,2,3") },
+
+    { njs_str("Array.prototype.join.call(new Uint8Array([0,1,2]))"),
+      njs_str("0,1,2") },
 
     { njs_str("var a = []; a[5] = 5; a"),
       njs_str(",,,,,5") },

From xeioex at nginx.com  Wed Jul  8 13:08:47 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Wed, 08 Jul 2020 13:08:47 +0000
Subject: [njs] HTTP: removed dead store assignment after 3df09cf5345c.
Message-ID: <hg.0c941c887ab7.1594213727.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/0c941c887ab7
branches:  
changeset: 1462:0c941c887ab7
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Wed Jul 08 13:07:06 2020 +0000
description:
HTTP: removed dead store assignment after 3df09cf5345c.

diffstat:

 nginx/ngx_http_js_module.c |  2 --
 1 files changed, 0 insertions(+), 2 deletions(-)

diffs (12 lines):

diff -r 960402554cc9 -r 0c941c887ab7 nginx/ngx_http_js_module.c
--- a/nginx/ngx_http_js_module.c	Wed Jul 08 13:07:05 2020 +0000
+++ b/nginx/ngx_http_js_module.c	Wed Jul 08 13:07:06 2020 +0000
@@ -1384,8 +1384,6 @@ ngx_http_js_header_generic(njs_vm_t *vm,
         length = 1;
     }
 
-    i = 0;
-
     for (i = 0; i < (ngx_uint_t) length; i++) {
         if (array != NULL) {
             setval = njs_vm_array_prop(vm, array, i, &lvalue);

From xeioex at nginx.com  Thu Jul  9 14:00:27 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 09 Jul 2020 14:00:27 +0000
Subject: [njs] Fixed SetFunctionName() with Symbol keys.
Message-ID: <hg.5f2034557bc3.1594303227.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/5f2034557bc3
branches:  
changeset: 1463:5f2034557bc3
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu Jul 09 13:57:10 2020 +0000
description:
Fixed SetFunctionName() with Symbol keys.

This improves 85c1b6ba326b.

diffstat:

 src/njs_builtin.c        |  27 +++++++++----
 src/njs_function.c       |  45 ++++++++++++++++++------
 src/njs_function.h       |   2 +-
 src/njs_json.c           |   4 +-
 src/njs_object.h         |   2 +-
 src/njs_object_prop.c    |   2 +-
 src/njs_string.c         |   2 +-
 src/njs_symbol.c         |  88 +++++++++++++++++++----------------------------
 src/njs_symbol.h         |   5 +-
 src/test/njs_unit_test.c |  10 +++++
 10 files changed, 106 insertions(+), 81 deletions(-)

diffs (377 lines):

diff -r 0c941c887ab7 -r 5f2034557bc3 src/njs_builtin.c
--- a/src/njs_builtin.c	Wed Jul 08 13:07:06 2020 +0000
+++ b/src/njs_builtin.c	Thu Jul 09 13:57:10 2020 +0000
@@ -396,6 +396,7 @@ njs_builtin_traverse(njs_vm_t *vm, njs_t
     u_char                  *p, *start, *end;
     njs_int_t               ret, n;
     njs_str_t               name;
+    njs_bool_t              symbol;
     njs_value_t             key;
     njs_function_t          *func;
     njs_object_prop_t       *prop;
@@ -431,29 +432,37 @@ njs_builtin_traverse(njs_vm_t *vm, njs_t
     end = buf + sizeof(buf);
 
     do {
+        symbol = 0;
         key = path[n]->prop->name;
 
         if (njs_slow_path(njs_is_symbol(&key))) {
-            ret = njs_symbol_to_string(vm, &key, &key, 1);
-            if (njs_slow_path(ret != NJS_OK)) {
-                name = njs_str_value("#BROKEN_KEY");
-            }
-
-        } else {
-            if (p != buf) {
-                *p++ = '.';
+            symbol = 1;
+            key = *njs_symbol_description(&key);
+            if (njs_is_undefined(&key)) {
+                key = njs_string_empty;
             }
         }
 
         njs_string_get(&key, &name);
 
-        if (njs_slow_path((p + name.length + 1) > end)) {
+        if (njs_slow_path((p + name.length + 3) > end)) {
             njs_type_error(vm, "njs_builtin_traverse() key is too long");
             return NJS_ERROR;
         }
 
+        if (symbol) {
+            *p++ = '[';
+
+        } else if (p != buf) {
+            *p++ = '.';
+        }
+
         p = njs_cpymem(p, name.start, name.length);
 
+        if (symbol) {
+            *p++ = ']';
+        }
+
     } while (n-- > 0);
 
     if (ctx->type == NJS_BUILTIN_TRAVERSE_MATCH) {
diff -r 0c941c887ab7 -r 5f2034557bc3 src/njs_function.c
--- a/src/njs_function.c	Wed Jul 08 13:07:06 2020 +0000
+++ b/src/njs_function.c	Thu Jul 09 13:57:10 2020 +0000
@@ -131,30 +131,53 @@ njs_function_active_closures(njs_vm_t *v
 
 njs_int_t
 njs_function_name_set(njs_vm_t *vm, njs_function_t *function,
-    njs_value_t *name, njs_bool_t bound)
+    njs_value_t *name, const char *prefix)
 {
-    u_char              *start;
+    u_char              *p;
+    size_t              len, symbol;
     njs_int_t           ret;
+    njs_value_t         value;
     njs_string_prop_t   string;
     njs_object_prop_t   *prop;
     njs_lvlhsh_query_t  lhq;
 
     prop = njs_object_prop_alloc(vm, &njs_string_name, name, 0);
-    if (njs_slow_path(name == NULL)) {
+    if (njs_slow_path(prop == NULL)) {
         return NJS_ERROR;
     }
 
-    if (bound) {
-        (void) njs_string_prop(&string, name);
+    symbol = 0;
+
+    if (njs_is_symbol(&prop->value)) {
+        symbol = 2;
+        prop->value = *njs_symbol_description(&prop->value);
+    }
 
-        start = njs_string_alloc(vm, &prop->value, string.size + 6,
-                                 string.length + 6);
-        if (njs_slow_path(start == NULL)) {
+    if (prefix != NULL || symbol != 0) {
+        value = prop->value;
+        (void) njs_string_prop(&string, &value);
+
+        len = (prefix != NULL) ? njs_strlen(prefix) + 1: 0;
+        p = njs_string_alloc(vm, &prop->value, string.size + len + symbol,
+                             string.length + len + symbol);
+        if (njs_slow_path(p == NULL)) {
             return NJS_ERROR;
         }
 
-        start = njs_cpymem(start, "bound ", 6);
-        memcpy(start, string.start, string.size);
+        if (len != 0) {
+            p = njs_cpymem(p, prefix, len - 1);
+            *p++ = ' ';
+        }
+
+        if (symbol != 0) {
+            *p++ = '[';
+        }
+
+        p = njs_cpymem(p, string.start, string.size);
+
+        if (symbol != 0) {
+            *p++ = ']';
+        }
     }
 
     prop->configurable = 1;
@@ -1204,7 +1227,7 @@ njs_function_prototype_bind(njs_vm_t *vm
         name = njs_string_empty;
     }
 
-    ret = njs_function_name_set(vm, function, &name, 1);
+    ret = njs_function_name_set(vm, function, &name, "bound");
     if (njs_slow_path(ret == NJS_ERROR)) {
         return ret;
     }
diff -r 0c941c887ab7 -r 5f2034557bc3 src/njs_function.h
--- a/src/njs_function.h	Wed Jul 08 13:07:06 2020 +0000
+++ b/src/njs_function.h	Thu Jul 09 13:57:10 2020 +0000
@@ -93,7 +93,7 @@ njs_function_t *njs_function_alloc(njs_v
     njs_closure_t *closures[], njs_bool_t shared);
 njs_function_t *njs_function_value_copy(njs_vm_t *vm, njs_value_t *value);
 njs_int_t njs_function_name_set(njs_vm_t *vm, njs_function_t *function,
-    njs_value_t *name, njs_bool_t bound);
+    njs_value_t *name, const char *prefix);
 njs_int_t njs_function_arguments_object_init(njs_vm_t *vm,
     njs_native_frame_t *frame);
 njs_int_t njs_function_rest_parameters_init(njs_vm_t *vm,
diff -r 0c941c887ab7 -r 5f2034557bc3 src/njs_json.c
--- a/src/njs_json.c	Wed Jul 08 13:07:06 2020 +0000
+++ b/src/njs_json.c	Thu Jul 09 13:57:10 2020 +0000
@@ -1856,7 +1856,7 @@ njs_dump_terminal(njs_json_stringify_t *
     case NJS_OBJECT_SYMBOL:
         value = njs_object_value(value);
 
-        ret = njs_symbol_to_string(stringify->vm, &str_val, value, 0);
+        ret = njs_symbol_descriptive_string(stringify->vm, &str_val, value);
         if (njs_slow_path(ret != NJS_OK)) {
             return NJS_ERROR;
         }
@@ -1867,7 +1867,7 @@ njs_dump_terminal(njs_json_stringify_t *
         break;
 
     case NJS_SYMBOL:
-        ret = njs_symbol_to_string(stringify->vm, &str_val, value, 0);
+        ret = njs_symbol_descriptive_string(stringify->vm, &str_val, value);
         if (njs_slow_path(ret != NJS_OK)) {
             return NJS_ERROR;
         }
diff -r 0c941c887ab7 -r 5f2034557bc3 src/njs_object.h
--- a/src/njs_object.h	Wed Jul 08 13:07:06 2020 +0000
+++ b/src/njs_object.h	Thu Jul 09 13:57:10 2020 +0000
@@ -213,7 +213,7 @@ njs_key_string_get(njs_vm_t *vm, njs_val
     njs_int_t  ret;
 
     if (njs_slow_path(njs_is_symbol(key))) {
-        ret = njs_symbol_to_string(vm, key, key, 0);
+        ret = njs_symbol_descriptive_string(vm, key, key);
         if (njs_slow_path(ret != NJS_OK)) {
             return ret;
         }
diff -r 0c941c887ab7 -r 5f2034557bc3 src/njs_object_prop.c
--- a/src/njs_object_prop.c	Wed Jul 08 13:07:06 2020 +0000
+++ b/src/njs_object_prop.c	Thu Jul 09 13:57:10 2020 +0000
@@ -554,7 +554,7 @@ njs_prop_private_copy(njs_vm_t *vm, njs_
             return NJS_ERROR;
         }
 
-        return njs_function_name_set(vm, function, &prop->name, 0);
+        return njs_function_name_set(vm, function, &prop->name, NULL);
 
     default:
         break;
diff -r 0c941c887ab7 -r 5f2034557bc3 src/njs_string.c
--- a/src/njs_string.c	Wed Jul 08 13:07:06 2020 +0000
+++ b/src/njs_string.c	Thu Jul 09 13:57:10 2020 +0000
@@ -486,7 +486,7 @@ njs_string_constructor(njs_vm_t *vm, njs
 
         if (njs_slow_path(!njs_is_string(value))) {
             if (!vm->top_frame->ctor && njs_is_symbol(value)) {
-                return njs_symbol_to_string(vm, &vm->retval, value, 0);
+                return njs_symbol_descriptive_string(vm, &vm->retval, value);
             }
 
             ret = njs_value_to_string(vm, value, value);
diff -r 0c941c887ab7 -r 5f2034557bc3 src/njs_symbol.c
--- a/src/njs_symbol.c	Wed Jul 08 13:07:06 2020 +0000
+++ b/src/njs_symbol.c	Thu Jul 09 13:57:10 2020 +0000
@@ -54,56 +54,52 @@ static const njs_value_t  *njs_symbol_na
 };
 
 
-njs_int_t
-njs_symbol_to_string(njs_vm_t *vm, njs_value_t *dst, const njs_value_t *value,
-    njs_bool_t as_name)
+const njs_value_t *
+njs_symbol_description(const njs_value_t *value)
 {
-    u_char             *start;
     const njs_value_t  *name;
-    njs_string_prop_t  string;
-
-    static const njs_value_t  string_symbol = njs_string("Symbol()");
 
-    name = value->data.u.value;
-
-    if (name == NULL) {
-        if (njs_fast_path(njs_symbol_key(value) < NJS_SYMBOL_KNOWN_MAX)) {
+    if (njs_symbol_key(value) < NJS_SYMBOL_KNOWN_MAX) {
+        name = njs_symbol_names[njs_symbol_key(value)];
 
-            name = njs_symbol_names[njs_symbol_key(value)];
+    } else {
+        name = value->data.u.value;
 
-        } else {
-            *dst = string_symbol;
-
-            return NJS_OK;
+        if (name == NULL) {
+            return  &njs_value_undefined;
         }
     }
 
-    (void) njs_string_prop(&string, name);
+    return name;
+}
 
-    if (as_name) {
-        string.length += njs_length("[]");
 
-        start = njs_string_alloc(vm, dst, string.size + 2, string.length);
-        if (njs_slow_path(start == NULL)) {
-            return NJS_ERROR;
-        }
+njs_int_t
+njs_symbol_descriptive_string(njs_vm_t *vm, njs_value_t *dst,
+    const njs_value_t *value)
+{
+    u_char             *start;
+    const njs_value_t  *description;
+    njs_string_prop_t  string;
+
+    description = njs_symbol_description(value);
 
-        start = njs_cpymem(start, "[", 1);
-        start = njs_cpymem(start, string.start, string.size);
-        *start = ']';
+    if (njs_is_undefined(description)) {
+        description = &njs_string_empty;
+    }
 
-    } else {
-        string.length += njs_length("Symbol()");
+    (void) njs_string_prop(&string, description);
+
+    string.length += njs_length("Symbol()");
 
-        start = njs_string_alloc(vm, dst, string.size + 8, string.length);
-        if (njs_slow_path(start == NULL)) {
-            return NJS_ERROR;
-        }
+    start = njs_string_alloc(vm, dst, string.size + 8, string.length);
+    if (njs_slow_path(start == NULL)) {
+        return NJS_ERROR;
+    }
 
-        start = njs_cpymem(start, "Symbol(", 7);
-        start = njs_cpymem(start, string.start, string.size);
-        *start = ')';
-    }
+    start = njs_cpymem(start, "Symbol(", 7);
+    start = njs_cpymem(start, string.start, string.size);
+    *start = ')';
 
     return NJS_OK;
 }
@@ -344,7 +340,7 @@ njs_symbol_prototype_to_string(njs_vm_t 
         return ret;
     }
 
-    return njs_symbol_to_string(vm, &vm->retval, &vm->retval, 0);
+    return njs_symbol_descriptive_string(vm, &vm->retval, &vm->retval);
 }
 
 
@@ -352,28 +348,14 @@ static njs_int_t
 njs_symbol_prototype_description(njs_vm_t *vm, njs_value_t *args,
     njs_uint_t nargs, njs_index_t unused)
 {
-    njs_int_t          ret;
-    const njs_value_t  *value, *name;
+    njs_int_t  ret;
 
     ret = njs_symbol_prototype_value_of(vm, args, nargs, unused);
     if (njs_slow_path(ret != NJS_OK)) {
         return ret;
     }
 
-    value = &vm->retval;
-
-    name = value->data.u.value;
-
-    if (name == NULL) {
-        if (njs_fast_path(njs_symbol_key(value) < NJS_SYMBOL_KNOWN_MAX)) {
-            name = njs_symbol_names[njs_symbol_key(value)];
-
-        } else {
-            name = &njs_value_undefined;
-        }
-    }
-
-    vm->retval = *name;
+    vm->retval = *njs_symbol_description(&vm->retval);
 
     return NJS_OK;
 }
diff -r 0c941c887ab7 -r 5f2034557bc3 src/njs_symbol.h
--- a/src/njs_symbol.h	Wed Jul 08 13:07:06 2020 +0000
+++ b/src/njs_symbol.h	Thu Jul 09 13:57:10 2020 +0000
@@ -7,8 +7,9 @@
 #ifndef _NJS_SYMBOL_H_INCLUDED_
 #define _NJS_SYMBOL_H_INCLUDED_
 
-njs_int_t njs_symbol_to_string(njs_vm_t *vm, njs_value_t *dst,
-    const njs_value_t *value, njs_bool_t as_name);
+const njs_value_t *njs_symbol_description(const njs_value_t *value);
+njs_int_t njs_symbol_descriptive_string(njs_vm_t *vm, njs_value_t *dst,
+    const njs_value_t *value);
 
 
 extern const njs_object_type_init_t  njs_symbol_type_init;
diff -r 0c941c887ab7 -r 5f2034557bc3 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Wed Jul 08 13:07:06 2020 +0000
+++ b/src/test/njs_unit_test.c	Thu Jul 09 13:57:10 2020 +0000
@@ -6698,6 +6698,13 @@ static njs_unit_test_t  njs_test[] =
               "[f.name, f.bind().name]"),
       njs_str("F,bound F") },
 
+    { njs_str("var s = Symbol('F'); var f = Object.defineProperty(function() {}, 'name', {get:()=>s});"
+              "[f.name.description, f.bind().name]"),
+      njs_str("F,bound ") },
+
+    { njs_str("/a/[Symbol.replace].bind().name"),
+      njs_str("bound [Symbol.replace]") },
+
     { njs_str("var f = Object.defineProperty(function() {}, 'name', {get:()=>{throw Error('Oops')}});"
               "f.bind().name"),
       njs_str("Error: Oops") },
@@ -7736,6 +7743,9 @@ static njs_unit_test_t  njs_test[] =
               "'ABC'.replace(/(B)/, '$<g>')"),
       njs_str("OOps") },
 
+    { njs_str("var name = /a/g[Symbol.replace].name; [name, typeof name]"),
+      njs_str("[Symbol.replace],string") },
+
     { njs_str("RegExp.prototype[Symbol.replace].call()"),
       njs_str("TypeError: \"this\" is not object") },
 

From gollam6 at gmail.com  Thu Jul  9 14:20:16 2020
From: gollam6 at gmail.com (Balazs Hinel)
Date: Thu, 09 Jul 2020 16:20:16 +0200
Subject: [PATCH] Stream: added extra variables for logging
Message-ID: <ab1359d9762109d1cccb.1594304416@ntasdev.emea.nsn-net.net>

# HG changeset patch
# User Balazs Hinel <balazs.hinel at nokia.com>
# Date 1594225255 -7200
#      Wed Jul 08 18:20:55 2020 +0200
# Node ID ab1359d9762109d1cccb06fbe85e17168b51ebed
# Parent  2ab0ecfe5a5f02f7214b02c8a1f9d6c5843d310e
Stream: added extra variables for logging.

As a result, the local and remote connection details (address and port)
used in the upstream connection can be logged.

diff -r 2ab0ecfe5a5f -r ab1359d97621 src/stream/ngx_stream.h
--- a/src/stream/ngx_stream.h	Tue Jul 07 18:56:06 2020 +0300
+++ b/src/stream/ngx_stream.h	Wed Jul 08 18:20:55 2020 +0200
@@ -192,6 +192,14 @@
 } ngx_stream_core_srv_conf_t;
 
 
+typedef struct {
+    ngx_str_t                      upstream_remote_addr;
+    in_port_t                      upstream_remote_port;
+    ngx_str_t                      upstream_local_addr;
+    in_port_t                      upstream_local_port;
+} ngx_stream_upstream_addr_t;
+
+
 struct ngx_stream_session_s {
     uint32_t                       signature;         /* "STRM" */
 
@@ -212,6 +220,8 @@
                                            /* of ngx_stream_upstream_state_t */
     ngx_stream_variable_value_t   *variables;
 
+    ngx_stream_upstream_addr_t     upstream_addrs;
+
 #if (NGX_PCRE)
     ngx_uint_t                     ncaptures;
     int                           *captures;
diff -r 2ab0ecfe5a5f -r ab1359d97621 src/stream/ngx_stream_proxy_module.c
--- a/src/stream/ngx_stream_proxy_module.c	Tue Jul 07 18:56:06 2020 +0300
+++ b/src/stream/ngx_stream_proxy_module.c	Wed Jul 08 18:20:55 2020 +0200
@@ -761,6 +761,8 @@
     ngx_stream_upstream_t        *u;
     ngx_stream_core_srv_conf_t   *cscf;
     ngx_stream_proxy_srv_conf_t  *pscf;
+    ngx_str_t                     local_sockaddr_str;
+    u_char                        addr[NGX_SOCKADDR_STRLEN];
 
     u = s->upstream;
     pc = u->peer.connection;
@@ -799,21 +801,42 @@
 
     c = s->connection;
 
-    if (c->log->log_level >= NGX_LOG_INFO) {
-        ngx_str_t  str;
-        u_char     addr[NGX_SOCKADDR_STRLEN];
-
-        str.len = NGX_SOCKADDR_STRLEN;
-        str.data = addr;
-
-        if (ngx_connection_local_sockaddr(pc, &str, 1) == NGX_OK) {
+    local_sockaddr_str.len = NGX_SOCKADDR_STRLEN;
+    local_sockaddr_str.data = addr;
+
+    if (ngx_connection_local_sockaddr(pc, &local_sockaddr_str, 1) == NGX_OK) {
+        s->upstream_addrs.upstream_remote_addr.data =
+                                              ngx_pnalloc(s->connection->pool,
+                                                          NGX_SOCKADDR_STRLEN);
+        s->upstream_addrs.upstream_remote_addr.len =
+                     ngx_sock_ntop(u->peer.sockaddr,
+                                   u->peer.socklen,
+                                   s->upstream_addrs.upstream_remote_addr.data,
+                                   NGX_SOCKADDR_STRLEN,
+                                   0);
+        s->upstream_addrs.upstream_remote_port =
+                                           ngx_inet_get_port(u->peer.sockaddr);
+
+        s->upstream_addrs.upstream_local_addr.data =
+                                              ngx_pnalloc(s->connection->pool,
+                                                          NGX_SOCKADDR_STRLEN);
+        s->upstream_addrs.upstream_local_addr.len =
+                      ngx_sock_ntop(pc->local_sockaddr,
+                                    pc->local_socklen,
+                                    s->upstream_addrs.upstream_local_addr.data,
+                                    NGX_SOCKADDR_STRLEN,
+                                    0);
+        s->upstream_addrs.upstream_local_port =
+                                         ngx_inet_get_port(pc->local_sockaddr);
+
+        if (c->log->log_level >= NGX_LOG_INFO) {
             handler = c->log->handler;
             c->log->handler = NULL;
 
             ngx_log_error(NGX_LOG_INFO, c->log, 0,
                           "%sproxy %V connected to %V",
                           pc->type == SOCK_DGRAM ? "udp " : "",
-                          &str, u->peer.name);
+                          &local_sockaddr_str, u->peer.name);
 
             c->log->handler = handler;
         }
diff -r 2ab0ecfe5a5f -r ab1359d97621 src/stream/ngx_stream_variables.c
--- a/src/stream/ngx_stream_variables.c	Tue Jul 07 18:56:06 2020 +0300
+++ b/src/stream/ngx_stream_variables.c	Wed Jul 08 18:20:55 2020 +0200
@@ -19,6 +19,14 @@
     ngx_stream_variable_value_t *v, uintptr_t data);
 static ngx_int_t ngx_stream_variable_remote_port(ngx_stream_session_t *s,
     ngx_stream_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_stream_variable_upstream_remote_addr(
+    ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_stream_variable_upstream_remote_port(
+    ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_stream_variable_upstream_local_addr(
+    ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_stream_variable_upstream_local_port(
+    ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data);
 static ngx_int_t ngx_stream_variable_proxy_protocol_addr(
     ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data);
 static ngx_int_t ngx_stream_variable_proxy_protocol_port(
@@ -63,6 +71,18 @@
     { ngx_string("remote_port"), NULL,
       ngx_stream_variable_remote_port, 0, 0, 0 },
 
+    { ngx_string("upstream_remote_addr"), NULL,
+      ngx_stream_variable_upstream_remote_addr, 0, 0, 0 },
+
+    { ngx_string("upstream_remote_port"), NULL,
+      ngx_stream_variable_upstream_remote_port, 0, 0, 0 },
+
+    { ngx_string("upstream_local_addr"), NULL,
+      ngx_stream_variable_upstream_local_addr, 0, 0, 0 },
+
+    { ngx_string("upstream_local_port"), NULL,
+      ngx_stream_variable_upstream_local_port, 0, 0, 0 },
+
     { ngx_string("proxy_protocol_addr"), NULL,
       ngx_stream_variable_proxy_protocol_addr,
       offsetof(ngx_proxy_protocol_t, src_addr), 0, 0 },
@@ -562,6 +582,85 @@
     return NGX_OK;
 }
 
+static ngx_int_t
+ngx_stream_variable_upstream_remote_addr(ngx_stream_session_t *s,
+    ngx_stream_variable_value_t *v, uintptr_t data)
+{
+    v->len = s->upstream_addrs.upstream_remote_addr.len;
+    v->valid = 1;
+    v->no_cacheable = 0;
+    v->not_found = 0;
+    v->data = s->upstream_addrs.upstream_remote_addr.data;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_stream_variable_upstream_remote_port(ngx_stream_session_t *s,
+    ngx_stream_variable_value_t *v, uintptr_t data)
+{
+    ngx_uint_t  port;
+
+    v->len = 0;
+    v->valid = 1;
+    v->no_cacheable = 0;
+    v->not_found = 0;
+
+    v->data = ngx_pnalloc(s->connection->pool, sizeof("65535") - 1);
+    if (v->data == NULL) {
+        return NGX_ERROR;
+    }
+
+    port = s->upstream_addrs.upstream_remote_port;
+
+    if (port > 0 && port < 65536) {
+        v->len = ngx_sprintf(v->data, "%ui", port) - v->data;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_stream_variable_upstream_local_addr(ngx_stream_session_t *s,
+    ngx_stream_variable_value_t *v, uintptr_t data)
+{
+    v->len = s->upstream_addrs.upstream_local_addr.len;
+    v->valid = 1;
+    v->no_cacheable = 0;
+    v->not_found = 0;
+    v->data = s->upstream_addrs.upstream_local_addr.data;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_stream_variable_upstream_local_port(ngx_stream_session_t *s,
+    ngx_stream_variable_value_t *v, uintptr_t data)
+{
+    ngx_uint_t  port;
+
+    v->len = 0;
+    v->valid = 1;
+    v->no_cacheable = 0;
+    v->not_found = 0;
+
+    v->data = ngx_pnalloc(s->connection->pool, sizeof("65535") - 1);
+    if (v->data == NULL) {
+        return NGX_ERROR;
+    }
+
+    port = s->upstream_addrs.upstream_local_port;
+
+    if (port > 0 && port < 65536) {
+        v->len = ngx_sprintf(v->data, "%ui", port) - v->data;
+    }
+
+    return NGX_OK;
+}
+
 
 static ngx_int_t
 ngx_stream_variable_proxy_protocol_addr(ngx_stream_session_t *s,

From arut at nginx.com  Thu Jul  9 14:43:26 2020
From: arut at nginx.com (Roman Arutyunyan)
Date: Thu, 09 Jul 2020 14:43:26 +0000
Subject: [nginx] Version bump.
Message-ID: <hg.fac6e1a46206.1594305806.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/fac6e1a46206
branches:  
changeset: 7683:fac6e1a46206
user:      Roman Arutyunyan <arut at nginx.com>
date:      Thu Jul 09 17:33:22 2020 +0300
description:
Version bump.

diffstat:

 src/core/nginx.h |  4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diffs (14 lines):

diff -r 2ab0ecfe5a5f -r fac6e1a46206 src/core/nginx.h
--- a/src/core/nginx.h	Tue Jul 07 18:56:06 2020 +0300
+++ b/src/core/nginx.h	Thu Jul 09 17:33:22 2020 +0300
@@ -9,8 +9,8 @@
 #define _NGINX_H_INCLUDED_
 
 
-#define nginx_version      1019001
-#define NGINX_VERSION      "1.19.1"
+#define nginx_version      1019002
+#define NGINX_VERSION      "1.19.2"
 #define NGINX_VER          "nginx/" NGINX_VERSION
 
 #ifdef NGX_BUILD

From arut at nginx.com  Thu Jul  9 14:43:29 2020
From: arut at nginx.com (Roman Arutyunyan)
Date: Thu, 09 Jul 2020 14:43:29 +0000
Subject: [nginx] Slice filter: clear original Accept-Ranges.
Message-ID: <hg.32a343635b50.1594305809.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/32a343635b50
branches:  
changeset: 7684:32a343635b50
user:      Roman Arutyunyan <arut at nginx.com>
date:      Thu Jul 09 16:21:37 2020 +0300
description:
Slice filter: clear original Accept-Ranges.

The slice filter allows ranges for the response by setting the r->allow_ranges
flag, which enables the range filter.  If the range was not requested, the
range filter adds an Accept-Ranges header to the response to signal the
support for ranges.

Previously, if an Accept-Ranges header was already present in the first slice
response, client received two copies of this header.  Now, the slice filter
removes the Accept-Ranges header from the response prior to setting the
r->allow_ranges flag.

diffstat:

 src/http/modules/ngx_http_slice_filter_module.c |  5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)

diffs (15 lines):

diff -r fac6e1a46206 -r 32a343635b50 src/http/modules/ngx_http_slice_filter_module.c
--- a/src/http/modules/ngx_http_slice_filter_module.c	Thu Jul 09 17:33:22 2020 +0300
+++ b/src/http/modules/ngx_http_slice_filter_module.c	Thu Jul 09 16:21:37 2020 +0300
@@ -180,6 +180,11 @@ ngx_http_slice_header_filter(ngx_http_re
     r->headers_out.content_range->hash = 0;
     r->headers_out.content_range = NULL;
 
+    if (r->headers_out.accept_ranges) {
+        r->headers_out.accept_ranges->hash = 0;
+        r->headers_out.accept_ranges = NULL;
+    }
+
     r->allow_ranges = 1;
     r->subrequest_ranges = 1;
     r->single_range = 1;

From vl at nginx.com  Fri Jul 10 14:03:10 2020
From: vl at nginx.com (Vladimir Homutov)
Date: Fri, 10 Jul 2020 17:03:10 +0300
Subject: [PATCH] Stream: added extra variables for logging
In-Reply-To: <ab1359d9762109d1cccb.1594304416@ntasdev.emea.nsn-net.net>
References: <ab1359d9762109d1cccb.1594304416@ntasdev.emea.nsn-net.net>
Message-ID: <20200710140310.GA59881@vlcx>

On Thu, Jul 09, 2020 at 04:20:16PM +0200, Balazs Hinel wrote:
> # HG changeset patch
> # User Balazs Hinel <balazs.hinel at nokia.com>
> # Date 1594225255 -7200
> #      Wed Jul 08 18:20:55 2020 +0200
> # Node ID ab1359d9762109d1cccb06fbe85e17168b51ebed
> # Parent  2ab0ecfe5a5f02f7214b02c8a1f9d6c5843d310e
> Stream: added extra variables for logging.
>
> As a result, the local and remote connection details (address and port)
> used in the upstream connection can be logged.
>
> diff -r 2ab0ecfe5a5f -r ab1359d97621 src/stream/ngx_stream.h
> --- a/src/stream/ngx_stream.h	Tue Jul 07 18:56:06 2020 +0300
> +++ b/src/stream/ngx_stream.h	Wed Jul 08 18:20:55 2020 +0200
> @@ -192,6 +192,14 @@
>  } ngx_stream_core_srv_conf_t;
>

Hello and thank you for contribution!

What problem are you trying to solve and why existing variables

http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_addr
http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_remote_addr
http://nginx.org/en/docs/stream/ngx_stream_realip_module.html#var_realip_remote_addr

are not enough?

Note also that we log connection events in the error log at INFO level in the
stream module.

From mdounin at mdounin.ru  Fri Jul 10 14:30:03 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Fri, 10 Jul 2020 17:30:03 +0300
Subject: Thrift proxy code contribution
In-Reply-To: <1594133067108.54868@didiglobal.com>
References: <1594132884832.70253@didiglobal.com>
 <1594133067108.54868@didiglobal.com>
Message-ID: <20200710143003.GT12747@mdounin.ru>

Hello!

On Tue, Jul 07, 2020 at 02:45:00PM +0000, ??? Junjie Wei wrote:

> Hi Maxim Dounin, I'm glad to receive your reply.
> 
> The nginx directories are as follow:
> 
> auto
> conf
> contrib
> docs
> misc
> objs
> src
> |-core
> |-event
> |-http
> |-mail
> |-misc
> |-os
> |-stream
> |-thrift
>   |-modules
> 
> 
> 
> Just like supporting tcp proxy, we developed a series of modules 
> to support thrift protocol proxying.
> 
> No matter using or compiling, these modules are optional.
> 
> 
> Config example:
> 
> worker_rlimit_nofile 204800;
> worker_processes 1;?
> error_log  logs/error.log error;
> pid        run/nginx.pid;
> http{
>     ...
> }
> thrift {
>     access_log logs/access.log;
> 
>     server {
>         listen 8000;
>         server_name counter; # counter is service name in thrift 
>         request
>         location ping {      # ping is method name in thrift 
>         request
>             proxy_pass ua;
>  ?       }
>     }
> 
>     upstream ua {
>         server 127.0.0.1:8003;
>     }
> }?
> 
> 
> Most of our codes are placed in thrift directory, which consists 
> of 1 core module:
> 
>   1.  ngx_thrift_core_module
> 
> and other functional modules:
> 
>   1.  ngx_thrift_upstream_module
> 
>   2.  ngx_thrift_proxy_module
>   3.  ngx_thrift_log_module
>   4.  ngx_thrift_upstream_check_module?tcp active health check
>   5.  ngx_thrift_upstream_keepalive_module
>   6.  ngx_http_dynamic_upstream_module : using http request to 
>   change nginx upstream endpoint without reload nginx conf
>   7.  ngx_thrift_limit_req_module
>   8.  ngx_thrift_dynamic_req_module: dispatching request to 
>   different upstream accoding to request content
> 
> The only things we change in original nginx are:
> 
>   1.  build script in auto directory to support compile thrift 
>   proxy core framework code into nginx
>   2.  macro constance like "#define NGX_LOG_DEBUG_THRIFT 0x800" 
>   in ngx_log.h, which is non-logical
> 
> Otherwise, we developed a standalone thrift procotol 
> encoder/decoder and we preliminarily plan to publish it 
> independently, which will be imported in nginx as a third party 
> lib.
> 
> Do you think out thoughts are feasible? We are really looking 
> forward for suggestions and opinions.

I don't see any problems with publishing your code as a module for 
nginx.  Changes you've mentioned in nginx itself should be 
relatively easy to avoid.

> By the way, must out codes support other OS like Windows or 
> other event mechanics like poll, select, kqueue?

That's completely up to you.  If you don't, it might be a good 
idea to outline somewhere in the documentation that your module 
has known bugs and cannot be used with some event methods.

-- 
Maxim Dounin
http://mdounin.ru/

From gollam6 at gmail.com  Fri Jul 10 16:10:44 2020
From: gollam6 at gmail.com (=?UTF-8?Q?Hinel_Bal=C3=A1zs?=)
Date: Fri, 10 Jul 2020 18:10:44 +0200
Subject: [PATCH] Stream: added extra variables for logging
In-Reply-To: <20200710140310.GA59881@vlcx>
References: <ab1359d9762109d1cccb.1594304416@ntasdev.emea.nsn-net.net>
 <20200710140310.GA59881@vlcx>
Message-ID: <CADzqfCnoUh6WJr-42asMo-wsT+s7N_Xy62o3kqTDsjNUWoEGaA@mail.gmail.com>

Hi,
The first one (upstream_addr) contains the address and port in one
single variable, but we need to put them separately into the access
log, that's why we need the two variables. The second one
(remote_addr) contains the client's address, which initiated the
connection from outside. The third one also contains the client
address, if I read correctly, and only if we would use the PROXY
protocol (we don't). What we added here are the variables for address
of the upstream, and the address of the internal interface of nginx,
which is used to establish the connection towards the upstream.

Best regards,
Balazs


On Fri, Jul 10, 2020 at 4:03 PM Vladimir Homutov <vl at nginx.com> wrote:
>
> On Thu, Jul 09, 2020 at 04:20:16PM +0200, Balazs Hinel wrote:
> > # HG changeset patch
> > # User Balazs Hinel <balazs.hinel at nokia.com>
> > # Date 1594225255 -7200
> > #      Wed Jul 08 18:20:55 2020 +0200
> > # Node ID ab1359d9762109d1cccb06fbe85e17168b51ebed
> > # Parent  2ab0ecfe5a5f02f7214b02c8a1f9d6c5843d310e
> > Stream: added extra variables for logging.
> >
> > As a result, the local and remote connection details (address and port)
> > used in the upstream connection can be logged.
> >
> > diff -r 2ab0ecfe5a5f -r ab1359d97621 src/stream/ngx_stream.h
> > --- a/src/stream/ngx_stream.h Tue Jul 07 18:56:06 2020 +0300
> > +++ b/src/stream/ngx_stream.h Wed Jul 08 18:20:55 2020 +0200
> > @@ -192,6 +192,14 @@
> >  } ngx_stream_core_srv_conf_t;
> >
>
> Hello and thank you for contribution!
>
> What problem are you trying to solve and why existing variables
>
> http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_addr
> http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_remote_addr
> http://nginx.org/en/docs/stream/ngx_stream_realip_module.html#var_realip_remote_addr
>
> are not enough?
>
> Note also that we log connection events in the error log at INFO level in the
> stream module.
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

From vl at nginx.com  Mon Jul 13 08:53:57 2020
From: vl at nginx.com (Vladimir Homutov)
Date: Mon, 13 Jul 2020 11:53:57 +0300
Subject: [PATCH] Stream: added extra variables for logging
In-Reply-To: <CADzqfCnoUh6WJr-42asMo-wsT+s7N_Xy62o3kqTDsjNUWoEGaA@mail.gmail.com>
References: <ab1359d9762109d1cccb.1594304416@ntasdev.emea.nsn-net.net>
 <20200710140310.GA59881@vlcx>
 <CADzqfCnoUh6WJr-42asMo-wsT+s7N_Xy62o3kqTDsjNUWoEGaA@mail.gmail.com>
Message-ID: <20200713085357.GA95029@vl.krasnogorsk.ru>

On Fri, Jul 10, 2020 at 06:10:44PM +0200, Hinel Bal?zs wrote:
> Hi,
> The first one (upstream_addr) contains the address and port in one
> single variable, but we need to put them separately into the access
> log, that's why we need the two variables. The second one
> (remote_addr) contains the client's address, which initiated the
> connection from outside. The third one also contains the client
> address, if I read correctly, and only if we would use the PROXY
> protocol (we don't). What we added here are the variables for address
> of the upstream, and the address of the internal interface of nginx,
> which is used to establish the connection towards the upstream.
>
> Best regards,
> Balazs

Note that it should be relatively easy to extract address/port into
separate variables using map or njs/other scripting language, so
you don't need to program new variable each time.

You may also want to look into implementation of the $upstream_addr
variable: it is located in the stream upstream module, and it is a
bit more complex than an addr:port pair, quoting the doc:

    If several servers were contacted during proxying, their addresses are
    separated by commas, e.g. ?192.168.1.1:12345, 192.168.1.2:12345, unix:/tmp/sock?.
    If a server cannot be selected, the variable keeps the name of the
    server group.

>
>
> On Fri, Jul 10, 2020 at 4:03 PM Vladimir Homutov <vl at nginx.com> wrote:
> >
> > On Thu, Jul 09, 2020 at 04:20:16PM +0200, Balazs Hinel wrote:
> > > # HG changeset patch
> > > # User Balazs Hinel <balazs.hinel at nokia.com>
> > > # Date 1594225255 -7200
> > > #      Wed Jul 08 18:20:55 2020 +0200
> > > # Node ID ab1359d9762109d1cccb06fbe85e17168b51ebed
> > > # Parent  2ab0ecfe5a5f02f7214b02c8a1f9d6c5843d310e
> > > Stream: added extra variables for logging.
> > >
> > > As a result, the local and remote connection details (address and port)
> > > used in the upstream connection can be logged.
> > >
> > > diff -r 2ab0ecfe5a5f -r ab1359d97621 src/stream/ngx_stream.h
> > > --- a/src/stream/ngx_stream.h Tue Jul 07 18:56:06 2020 +0300
> > > +++ b/src/stream/ngx_stream.h Wed Jul 08 18:20:55 2020 +0200
> > > @@ -192,6 +192,14 @@
> > >  } ngx_stream_core_srv_conf_t;
> > >
> >
> > Hello and thank you for contribution!
> >
> > What problem are you trying to solve and why existing variables
> >
> > http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_addr
> > http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_remote_addr
> > http://nginx.org/en/docs/stream/ngx_stream_realip_module.html#var_realip_remote_addr
> >
> > are not enough?
> >
> > Note also that we log connection events in the error log at INFO level in the
> > stream module.
> > _______________________________________________
> > nginx-devel mailing list
> > nginx-devel at nginx.org
> > http://mailman.nginx.org/mailman/listinfo/nginx-devel
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

From gollam6 at gmail.com  Mon Jul 13 09:00:55 2020
From: gollam6 at gmail.com (=?UTF-8?Q?Hinel_Bal=C3=A1zs?=)
Date: Mon, 13 Jul 2020 11:00:55 +0200
Subject: [PATCH] Stream: added extra variables for logging
In-Reply-To: <20200713085357.GA95029@vl.krasnogorsk.ru>
References: <ab1359d9762109d1cccb.1594304416@ntasdev.emea.nsn-net.net>
 <20200710140310.GA59881@vlcx>
 <CADzqfCnoUh6WJr-42asMo-wsT+s7N_Xy62o3kqTDsjNUWoEGaA@mail.gmail.com>
 <20200713085357.GA95029@vl.krasnogorsk.ru>
Message-ID: <CADzqfCmuBn5yPAzTOT9kTR14uzO5x4r7s9KBOA-fT9ekiVABZA@mail.gmail.com>

Ok, I agree on that one. How about the other one (internal interface
address of nginx)?

On Mon, Jul 13, 2020 at 10:54 AM Vladimir Homutov <vl at nginx.com> wrote:
>
> On Fri, Jul 10, 2020 at 06:10:44PM +0200, Hinel Bal?zs wrote:
> > Hi,
> > The first one (upstream_addr) contains the address and port in one
> > single variable, but we need to put them separately into the access
> > log, that's why we need the two variables. The second one
> > (remote_addr) contains the client's address, which initiated the
> > connection from outside. The third one also contains the client
> > address, if I read correctly, and only if we would use the PROXY
> > protocol (we don't). What we added here are the variables for address
> > of the upstream, and the address of the internal interface of nginx,
> > which is used to establish the connection towards the upstream.
> >
> > Best regards,
> > Balazs
>
> Note that it should be relatively easy to extract address/port into
> separate variables using map or njs/other scripting language, so
> you don't need to program new variable each time.
>
> You may also want to look into implementation of the $upstream_addr
> variable: it is located in the stream upstream module, and it is a
> bit more complex than an addr:port pair, quoting the doc:
>
>     If several servers were contacted during proxying, their addresses are
>     separated by commas, e.g. ?192.168.1.1:12345, 192.168.1.2:12345, unix:/tmp/sock?.
>     If a server cannot be selected, the variable keeps the name of the
>     server group.
>
> >
> >
> > On Fri, Jul 10, 2020 at 4:03 PM Vladimir Homutov <vl at nginx.com> wrote:
> > >
> > > On Thu, Jul 09, 2020 at 04:20:16PM +0200, Balazs Hinel wrote:
> > > > # HG changeset patch
> > > > # User Balazs Hinel <balazs.hinel at nokia.com>
> > > > # Date 1594225255 -7200
> > > > #      Wed Jul 08 18:20:55 2020 +0200
> > > > # Node ID ab1359d9762109d1cccb06fbe85e17168b51ebed
> > > > # Parent  2ab0ecfe5a5f02f7214b02c8a1f9d6c5843d310e
> > > > Stream: added extra variables for logging.
> > > >
> > > > As a result, the local and remote connection details (address and port)
> > > > used in the upstream connection can be logged.
> > > >
> > > > diff -r 2ab0ecfe5a5f -r ab1359d97621 src/stream/ngx_stream.h
> > > > --- a/src/stream/ngx_stream.h Tue Jul 07 18:56:06 2020 +0300
> > > > +++ b/src/stream/ngx_stream.h Wed Jul 08 18:20:55 2020 +0200
> > > > @@ -192,6 +192,14 @@
> > > >  } ngx_stream_core_srv_conf_t;
> > > >
> > >
> > > Hello and thank you for contribution!
> > >
> > > What problem are you trying to solve and why existing variables
> > >
> > > http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_addr
> > > http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_remote_addr
> > > http://nginx.org/en/docs/stream/ngx_stream_realip_module.html#var_realip_remote_addr
> > >
> > > are not enough?
> > >
> > > Note also that we log connection events in the error log at INFO level in the
> > > stream module.
> > > _______________________________________________
> > > nginx-devel mailing list
> > > nginx-devel at nginx.org
> > > http://mailman.nginx.org/mailman/listinfo/nginx-devel
> > _______________________________________________
> > nginx-devel mailing list
> > nginx-devel at nginx.org
> > http://mailman.nginx.org/mailman/listinfo/nginx-devel
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

From vl at nginx.com  Mon Jul 13 09:06:28 2020
From: vl at nginx.com (Vladimir Homutov)
Date: Mon, 13 Jul 2020 12:06:28 +0300
Subject: [PATCH] Stream: added extra variables for logging
In-Reply-To: <CADzqfCmuBn5yPAzTOT9kTR14uzO5x4r7s9KBOA-fT9ekiVABZA@mail.gmail.com>
References: <ab1359d9762109d1cccb.1594304416@ntasdev.emea.nsn-net.net>
 <20200710140310.GA59881@vlcx>
 <CADzqfCnoUh6WJr-42asMo-wsT+s7N_Xy62o3kqTDsjNUWoEGaA@mail.gmail.com>
 <20200713085357.GA95029@vl.krasnogorsk.ru>
 <CADzqfCmuBn5yPAzTOT9kTR14uzO5x4r7s9KBOA-fT9ekiVABZA@mail.gmail.com>
Message-ID: <18b113ad-a8aa-de8e-76d6-0c74ea35174f@nginx.com>

13.07.2020 12:00, Hinel Bal?zs ?????:
> Ok, I agree on that one. How about the other one (internal interface
> address of nginx)?

same logic applies. If nginx failed to connect to one upstream server,
it will try next one, and local port (and address maybe too) will change.


> 
> On Mon, Jul 13, 2020 at 10:54 AM Vladimir Homutov <vl at nginx.com> wrote:
>>
>> On Fri, Jul 10, 2020 at 06:10:44PM +0200, Hinel Bal?zs wrote:
>>> Hi,
>>> The first one (upstream_addr) contains the address and port in one
>>> single variable, but we need to put them separately into the access
>>> log, that's why we need the two variables. The second one
>>> (remote_addr) contains the client's address, which initiated the
>>> connection from outside. The third one also contains the client
>>> address, if I read correctly, and only if we would use the PROXY
>>> protocol (we don't). What we added here are the variables for address
>>> of the upstream, and the address of the internal interface of nginx,
>>> which is used to establish the connection towards the upstream.
>>>
>>> Best regards,
>>> Balazs
>>
>> Note that it should be relatively easy to extract address/port into
>> separate variables using map or njs/other scripting language, so
>> you don't need to program new variable each time.
>>
>> You may also want to look into implementation of the $upstream_addr
>> variable: it is located in the stream upstream module, and it is a
>> bit more complex than an addr:port pair, quoting the doc:
>>
>>      If several servers were contacted during proxying, their addresses are
>>      separated by commas, e.g. ?192.168.1.1:12345, 192.168.1.2:12345, unix:/tmp/sock?.
>>      If a server cannot be selected, the variable keeps the name of the
>>      server group.
>>
>>>
>>>
>>> On Fri, Jul 10, 2020 at 4:03 PM Vladimir Homutov <vl at nginx.com> wrote:
>>>>
>>>> On Thu, Jul 09, 2020 at 04:20:16PM +0200, Balazs Hinel wrote:
>>>>> # HG changeset patch
>>>>> # User Balazs Hinel <balazs.hinel at nokia.com>
>>>>> # Date 1594225255 -7200
>>>>> #      Wed Jul 08 18:20:55 2020 +0200
>>>>> # Node ID ab1359d9762109d1cccb06fbe85e17168b51ebed
>>>>> # Parent  2ab0ecfe5a5f02f7214b02c8a1f9d6c5843d310e
>>>>> Stream: added extra variables for logging.
>>>>>
>>>>> As a result, the local and remote connection details (address and port)
>>>>> used in the upstream connection can be logged.
>>>>>
>>>>> diff -r 2ab0ecfe5a5f -r ab1359d97621 src/stream/ngx_stream.h
>>>>> --- a/src/stream/ngx_stream.h Tue Jul 07 18:56:06 2020 +0300
>>>>> +++ b/src/stream/ngx_stream.h Wed Jul 08 18:20:55 2020 +0200
>>>>> @@ -192,6 +192,14 @@
>>>>>   } ngx_stream_core_srv_conf_t;
>>>>>
>>>>
>>>> Hello and thank you for contribution!
>>>>
>>>> What problem are you trying to solve and why existing variables
>>>>
>>>> http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_addr
>>>> http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_remote_addr
>>>> http://nginx.org/en/docs/stream/ngx_stream_realip_module.html#var_realip_remote_addr
>>>>
>>>> are not enough?
>>>>
>>>> Note also that we log connection events in the error log at INFO level in the
>>>> stream module.
>>>> _______________________________________________
>>>> nginx-devel mailing list
>>>> nginx-devel at nginx.org
>>>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>>> _______________________________________________
>>> nginx-devel mailing list
>>> nginx-devel at nginx.org
>>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>> _______________________________________________
>> nginx-devel mailing list
>> nginx-devel at nginx.org
>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel
> 


From ihaitao at qq.com  Tue Jul 14 03:41:03 2020
From: ihaitao at qq.com (=?utf-8?B?5ZCV5rW35rab?=)
Date: Tue, 14 Jul 2020 11:41:03 +0800
Subject: HTTP: support http forward proxying
Message-ID: <mailman.0.1594698086.2911.nginx-devel@nginx.org>

Hello, nginx,

This is my first try for adding the http forward proxying feature to nginx.

The http forward proxy protocol runs on plain http connection, which has big privacy issue.
So web browsers like chrome and firefox has added support for forward proxy over https.

We use nginx as https server, and we also need the forward proxy feature over https on the same host.
Nginx does not support forward proxying now. If we launch another security http proxy server, we
have no choice but stop the nginx server, because both the proxy server and nginx will listen on the
same 443 port.

So, the http forward proxying is need for nginx for us.

After investigate the nginx source, I found that almost all features needed for http forward proxying
are there. If we implement this feature as a dedicate module, we have to duplicate too many code
from the http_proxy_module and http_upstream_module. So I propose to make small modification to the
exists code base.

I use this implementation for weeks and it works fine.

If you want to test this feature, you need add the following conf to your *default* server.

    server {
        listen       443 ssl;

        # ssl conf

        location / {
            resolver 8.8.8.8;
            if ($host != $server_name) {
                auth_basic "forward proxy auth";
                auth_basic_user_file /path/to/htpasswd;

                proxy_pass forward;
            }
            try_files $uri $uri/ =404;
        }
    }

The current implementation does not support proxying over http2. As we cannot set the Host header to
the virtual server name, it is impossible to support forward proxying on non-default server, as well.

Here is the patch. Please make your comment. Thank you. 


commit 89877712df3e769f4d0acb8fdcefcd351b31eaad
Author: ??? <ihaitao at qq.com>
Date:   Sat Jun 6 12:16:42 2020 +0800

    support http forward proxy

diff --git a/auto/configure b/auto/configure
index 7e6e33a7..1ee0bfaa 100755
--- a/auto/configure
+++ b/auto/configure
@@ -92,6 +92,9 @@ have=NGX_HTTP_CLIENT_TEMP_PATH value="\"$NGX_HTTP_CLIENT_TEMP_PATH\""
 . auto/define
 have=NGX_HTTP_PROXY_TEMP_PATH value="\"$NGX_HTTP_PROXY_TEMP_PATH\""
 . auto/define
+if [ "$HTTP_PROXY_FORWARD" = "YES" ]; then
+    have=NGX_HTTP_PROXY_FORWARD . auto/have
+fi
 have=NGX_HTTP_FASTCGI_TEMP_PATH value="\"$NGX_HTTP_FASTCGI_TEMP_PATH\""
 . auto/define
 have=NGX_HTTP_UWSGI_TEMP_PATH value="\"$NGX_HTTP_UWSGI_TEMP_PATH\""
diff --git a/auto/options b/auto/options
index 521c9768..833ac06f 100644
--- a/auto/options
+++ b/auto/options
@@ -82,6 +82,7 @@ HTTP_SPLIT_CLIENTS=YES
 HTTP_REFERER=YES
 HTTP_REWRITE=YES
 HTTP_PROXY=YES
+HTTP_PROXY_FORWARD=NO
 HTTP_FASTCGI=YES
 HTTP_UWSGI=YES
 HTTP_SCGI=YES
@@ -245,6 +246,7 @@ $0: warning: the \"--with-ipv6\" option is deprecated"
         --with-http_secure_link_module)  HTTP_SECURE_LINK=YES       ;;
         --with-http_degradation_module)  HTTP_DEGRADATION=YES       ;;
         --with-http_slice_module)        HTTP_SLICE=YES             ;;
+        --with-http_proxy_forward)       HTTP_PROXY_FORWARD=YES     ;;
 
         --without-http_charset_module)   HTTP_CHARSET=NO            ;;
         --without-http_gzip_module)      HTTP_GZIP=NO               ;;
@@ -460,6 +462,7 @@ cat << END
   --with-http_degradation_module     enable ngx_http_degradation_module
   --with-http_slice_module           enable ngx_http_slice_module
   --with-http_stub_status_module     enable ngx_http_stub_status_module
+  --with-http_proxy_forward          enable http forward proxy support
 
   --without-http_charset_module      disable ngx_http_charset_module
   --without-http_gzip_module         disable ngx_http_gzip_module
diff --git a/src/http/modules/ngx_http_auth_basic_module.c b/src/http/modules/ngx_http_auth_basic_module.c
index ed9df343..39914e2f 100644
--- a/src/http/modules/ngx_http_auth_basic_module.c
+++ b/src/http/modules/ngx_http_auth_basic_module.c
@@ -36,7 +36,7 @@ static char *ngx_http_auth_basic_user_file(ngx_conf_t *cf, ngx_command_t *cmd,
 static ngx_command_t  ngx_http_auth_basic_commands[] = {
 
     { ngx_string("auth_basic"),
-      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LMT_CONF
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_HTTP_LMT_CONF
                         |NGX_CONF_TAKE1,
       ngx_http_set_complex_value_slot,
       NGX_HTTP_LOC_CONF_OFFSET,
@@ -44,7 +44,7 @@ static ngx_command_t  ngx_http_auth_basic_commands[] = {
       NULL },
 
     { ngx_string("auth_basic_user_file"),
-      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LMT_CONF
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_HTTP_LMT_CONF
                         |NGX_CONF_TAKE1,
       ngx_http_auth_basic_user_file,
       NGX_HTTP_LOC_CONF_OFFSET,
@@ -343,6 +343,13 @@ ngx_http_auth_basic_set_realm(ngx_http_request_t *r, ngx_str_t *realm)
     r->headers_out.www_authenticate->value.data = basic;
     r->headers_out.www_authenticate->value.len = len;
 
+#if (NGX_HTTP_PROXY_FORWARD)
+    if (r->method == NGX_HTTP_CONNECT) {
+	    ngx_str_set(&r->headers_out.www_authenticate->key, "Proxy-Authenticate");
+	    return NGX_HTTP_PROXY_AUTH_REQUIRED;
+    }
+#endif
+
     return NGX_HTTP_UNAUTHORIZED;
 }
 
diff --git a/src/http/modules/ngx_http_chunked_filter_module.c b/src/http/modules/ngx_http_chunked_filter_module.c
index 4d6fd3ee..50ab3513 100644
--- a/src/http/modules/ngx_http_chunked_filter_module.c
+++ b/src/http/modules/ngx_http_chunked_filter_module.c
@@ -77,6 +77,9 @@ ngx_http_chunked_header_filter(ngx_http_request_t *r)
         clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
 
         if (r->http_version >= NGX_HTTP_VERSION_11
+#if (NGX_HTTP_PROXY_FORWARD)
+            && r->method != NGX_HTTP_CONNECT
+#endif
             && clcf->chunked_transfer_encoding)
         {
             if (r->expect_trailers) {
diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
index 6cf2cbde..389f3e05 100644
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -84,6 +84,9 @@ typedef struct {
     ngx_http_proxy_vars_t          vars;
 
     ngx_flag_t                     redirect;
+#if (NGX_HTTP_PROXY_FORWARD)
+    ngx_flag_t                     forward;
+#endif
 
     ngx_uint_t                     http_version;
 
@@ -125,6 +128,7 @@ static ngx_int_t ngx_http_proxy_eval(ngx_http_request_t *r,
 static ngx_int_t ngx_http_proxy_create_key(ngx_http_request_t *r);
 #endif
 static ngx_int_t ngx_http_proxy_create_request(ngx_http_request_t *r);
+static ngx_int_t ngx_http_proxy_create_empty_request(ngx_http_request_t *r);
 static ngx_int_t ngx_http_proxy_reinit_request(ngx_http_request_t *r);
 static ngx_int_t ngx_http_proxy_body_output_filter(void *data, ngx_chain_t *in);
 static ngx_int_t ngx_http_proxy_process_status_line(ngx_http_request_t *r);
@@ -226,7 +230,6 @@ static ngx_conf_bitmask_t  ngx_http_proxy_next_upstream_masks[] = {
     { ngx_null_string, 0 }
 };
 
-
 #if (NGX_HTTP_SSL)
 
 static ngx_conf_bitmask_t  ngx_http_proxy_ssl_protocols[] = {
@@ -926,6 +929,19 @@ ngx_http_proxy_handler(ngx_http_request_t *r)
 
     u->accel = 1;
 
+#if (NGX_HTTP_PROXY_FORWARD)
+    if (plcf->forward && r->method == NGX_HTTP_CONNECT) {
+	    u->forward = 1;
+	    u->create_request = ngx_http_proxy_create_empty_request;
+
+	    r->main->count++;
+
+	    ngx_http_upstream_init(r);
+
+	    return NGX_DONE;
+    }
+#endif
+
     if (!plcf->upstream.request_buffering
         && plcf->body_values == NULL && plcf->upstream.pass_request_body
         && (!r->headers_in.chunked
@@ -1145,6 +1161,16 @@ ngx_http_proxy_create_key(ngx_http_request_t *r)
 
 #endif
 
+#if (NGX_HTTP_PROXY_FORWARD)
+static ngx_int_t
+ngx_http_proxy_create_empty_request(ngx_http_request_t *r)
+{
+    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "create empty http proxy request");
+
+    return NGX_OK;
+}
+#endif
 
 static ngx_int_t
 ngx_http_proxy_create_request(ngx_http_request_t *r)
@@ -2885,6 +2911,7 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf)
      *     conf->body_values = NULL;
      *     conf->body_source = { 0, NULL };
      *     conf->redirects = NULL;
+     *     conf->forward = 0;
      *     conf->ssl = 0;
      *     conf->ssl_protocols = 0;
      *     conf->ssl_ciphers = { 0, NULL };
@@ -3646,6 +3673,11 @@ ngx_http_proxy_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
     ngx_http_core_loc_conf_t   *clcf;
     ngx_http_script_compile_t   sc;
 
+#if (NGX_HTTP_PROXY_FORWARD)
+    static ngx_str_t forward_proxy_url =
+        ngx_string("http://$http_host$uri$is_args$args");
+#endif
+
     if (plcf->upstream.upstream || plcf->proxy_lengths) {
         return "is duplicate";
     }
@@ -3662,6 +3694,13 @@ ngx_http_proxy_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
 
     url = &value[1];
 
+#if (NGX_HTTP_PROXY_FORWARD)
+    if (ngx_strncasecmp(url->data, (u_char *) "forward", 7) == 0) {
+        plcf->forward = 1;
+        url = &forward_proxy_url;
+    }
+#endif
+
     n = ngx_http_script_variables_count(url);
 
     if (n) {
diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c
index 3671558d..c0638579 100644
--- a/src/http/ngx_http_core_module.c
+++ b/src/http/ngx_http_core_module.c
@@ -1938,13 +1938,17 @@ ngx_http_auth_basic_user(ngx_http_request_t *r)
         return NGX_DECLINED;
     }
 
-    if (r->headers_in.authorization == NULL) {
+    if (r->headers_in.authorization != NULL) {
+        encoded = r->headers_in.authorization->value;
+#if (NGX_HTTP_PROXY_FORWARD)
+    } else if (r->headers_in.proxy_authorization != NULL) {
+        encoded = r->headers_in.proxy_authorization->value;
+#endif
+    } else {
         r->headers_in.user.data = (u_char *) "";
         return NGX_DECLINED;
     }
 
-    encoded = r->headers_in.authorization->value;
-
     if (encoded.len < sizeof("Basic ") - 1
         || ngx_strncasecmp(encoded.data, (u_char *) "Basic ",
                            sizeof("Basic ") - 1)
@@ -4420,6 +4424,9 @@ static ngx_http_method_name_t  ngx_methods_names[] = {
     { (u_char *) "LOCK",      (uint32_t) ~NGX_HTTP_LOCK },
     { (u_char *) "UNLOCK",    (uint32_t) ~NGX_HTTP_UNLOCK },
     { (u_char *) "PATCH",     (uint32_t) ~NGX_HTTP_PATCH },
+#if (NGX_HTTP_PROXY_FORWARD)
+    { (u_char *) "CONNECT",   (uint32_t) ~NGX_HTTP_CONNECT },
+#endif
     { NULL, 0 }
 };
 
diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c
index cfc42f9d..f2eabd23 100644
--- a/src/http/ngx_http_parse.c
+++ b/src/http/ngx_http_parse.c
@@ -108,6 +108,9 @@ ngx_http_parse_request_line(ngx_http_request_t *r, ngx_buf_t *b)
         sw_start = 0,
         sw_method,
         sw_spaces_before_uri,
+#if (NGX_HTTP_PROXY_FORWARD)
+        sw_spaces_after_connect,
+#endif
         sw_schema,
         sw_schema_slash,
         sw_schema_slash_slash,
@@ -241,6 +244,14 @@ ngx_http_parse_request_line(ngx_http_request_t *r, ngx_buf_t *b)
                     break;
 
                 case 7:
+#if (NGX_HTTP_PROXY_FORWARD)
+                    if (ngx_str7_cmp(m, 'C', 'O', 'N', 'N', 'E', 'C', 'T', ' '))
+                    {
+                        r->method = NGX_HTTP_CONNECT;
+                        break;
+                    }
+#endif
+
                     if (ngx_str7_cmp(m, 'O', 'P', 'T', 'I', 'O', 'N', 'S', ' '))
                     {
                         r->method = NGX_HTTP_OPTIONS;
@@ -267,6 +278,13 @@ ngx_http_parse_request_line(ngx_http_request_t *r, ngx_buf_t *b)
                 }
 
                 state = sw_spaces_before_uri;
+
+#if (NGX_HTTP_PROXY_FORWARD)
+                if (r->method == NGX_HTTP_CONNECT) {
+                    state = sw_spaces_after_connect;
+                }
+#endif
+
                 break;
             }
 
@@ -342,6 +360,19 @@ ngx_http_parse_request_line(ngx_http_request_t *r, ngx_buf_t *b)
             }
             break;
 
+#if (NGX_HTTP_PROXY_FORWARD)
+        /* space* after CONNECT method */
+        case sw_spaces_after_connect:
+
+            if (ch == ' ') {
+                break;
+            }
+
+            state = sw_host_start;
+
+            /* fall through */
+#endif
+
         case sw_host_start:
 
             r->host_start = p;
@@ -715,6 +746,13 @@ ngx_http_parse_request_line(ngx_http_request_t *r, ngx_buf_t *b)
             switch (ch) {
             case '/':
                 state = sw_first_major_digit;
+#if (NGX_HTTP_PROXY_FORWARD)
+                if (r->method == NGX_HTTP_CONNECT) {
+                    r->uri_start = p;
+                    r->uri_end = p + 1;
+                }
+#endif
+
                 break;
             default:
                 return NGX_HTTP_PARSE_INVALID_REQUEST;
@@ -838,6 +876,13 @@ done:
         return NGX_HTTP_PARSE_INVALID_09_METHOD;
     }
 
+#if (NGX_HTTP_PROXY_FORWARD)
+    if (r->http_version < NGX_HTTP_VERSION_11 && r->method == NGX_HTTP_CONNECT)
+    {
+        return NGX_HTTP_PARSE_INVALID_METHOD;
+    }
+#endif
+
     return NGX_OK;
 }
 
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
index 6feb6cc3..8cf24298 100644
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -158,6 +158,12 @@ ngx_http_header_t  ngx_http_headers_in[] = {
                  offsetof(ngx_http_headers_in_t, authorization),
                  ngx_http_process_unique_header_line },
 
+#if (NGX_HTTP_PROXY_FORWARD)
+    { ngx_string("Proxy-Authorization"),
+                 offsetof(ngx_http_headers_in_t, proxy_authorization),
+                 ngx_http_process_unique_header_line },
+#endif
+
     { ngx_string("Keep-Alive"), offsetof(ngx_http_headers_in_t, keep_alive),
                  ngx_http_process_header_line },
 
diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h
index 70c2d424..0b04f21c 100644
--- a/src/http/ngx_http_request.h
+++ b/src/http/ngx_http_request.h
@@ -41,6 +41,7 @@
 #define NGX_HTTP_UNLOCK                    0x2000
 #define NGX_HTTP_PATCH                     0x4000
 #define NGX_HTTP_TRACE                     0x8000
+#define NGX_HTTP_CONNECT                   0x10000
 
 #define NGX_HTTP_CONNECTION_CLOSE          1
 #define NGX_HTTP_CONNECTION_KEEP_ALIVE     2
@@ -92,6 +93,7 @@
 #define NGX_HTTP_FORBIDDEN                 403
 #define NGX_HTTP_NOT_FOUND                 404
 #define NGX_HTTP_NOT_ALLOWED               405
+#define NGX_HTTP_PROXY_AUTH_REQUIRED       407
 #define NGX_HTTP_REQUEST_TIME_OUT          408
 #define NGX_HTTP_CONFLICT                  409
 #define NGX_HTTP_LENGTH_REQUIRED           411
@@ -207,6 +209,9 @@ typedef struct {
 #endif
 
     ngx_table_elt_t                  *authorization;
+#if (NGX_HTTP_PROXY_FORWARD)
+    ngx_table_elt_t                  *proxy_authorization;
+#endif
 
     ngx_table_elt_t                  *keep_alive;
 
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
index 47f98ccb..9b227adf 100644
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -45,6 +45,10 @@ static ngx_int_t ngx_http_upstream_send_request_body(ngx_http_request_t *r,
     ngx_http_upstream_t *u, ngx_uint_t do_write);
 static void ngx_http_upstream_send_request_handler(ngx_http_request_t *r,
     ngx_http_upstream_t *u);
+#if (NGX_HTTP_PROXY_FORWARD)
+static void ngx_http_upstream_send_connected_handler(ngx_http_request_t *r,
+    ngx_http_upstream_t *u);
+#endif
 static void ngx_http_upstream_read_request_handler(ngx_http_request_t *r);
 static void ngx_http_upstream_process_header(ngx_http_request_t *r,
     ngx_http_upstream_t *u);
@@ -1560,6 +1564,12 @@ ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u)
     u->write_event_handler = ngx_http_upstream_send_request_handler;
     u->read_event_handler = ngx_http_upstream_process_header;
 
+#if (NGX_HTTP_PROXY_FORWARD)
+    if (u->forward) {
+	    u->write_event_handler = ngx_http_upstream_send_connected_handler;
+    }
+#endif
+
     c->sendfile &= r->connection->sendfile;
     u->output.sendfile = c->sendfile;
 
@@ -1643,6 +1653,13 @@ ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u)
 
 #endif
 
+#if (NGX_HTTP_PROXY_FORWARD)
+    if (u->forward) {
+	    ngx_http_upstream_send_connected_handler(r, u);
+	    return;
+    }
+#endif
+
     ngx_http_upstream_send_request(r, u, 1);
 }
 
@@ -2255,6 +2272,40 @@ ngx_http_upstream_send_request_handler(ngx_http_request_t *r,
     ngx_http_upstream_send_request(r, u, 1);
 }
 
+#if (NGX_HTTP_PROXY_FORWARD)
+static void
+ngx_http_upstream_send_connected_handler(ngx_http_request_t *r,
+    ngx_http_upstream_t *u)
+{
+    ngx_connection_t  *c;
+
+    if (u->header_sent) {
+	    return;
+    }
+
+    c = u->peer.connection;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "http upstream send connected handler");
+
+    if (c->write->timedout) {
+        ngx_http_upstream_finalize_request(r, u, NGX_HTTP_BAD_GATEWAY);
+        return;
+    }
+
+    if (u->state->connect_time == (ngx_msec_t) -1) {
+        u->state->connect_time = ngx_current_msec - u->start_time;
+    }
+
+    r->headers_out.status = NGX_HTTP_OK;
+    r->chunked = 0;
+
+    u->upgrade = 1;
+
+    ngx_http_upstream_send_response(r, u);
+}
+#endif
+
 
 static void
 ngx_http_upstream_read_request_handler(ngx_http_request_t *r)
@@ -3377,19 +3428,20 @@ ngx_http_upstream_process_upgraded(ngx_http_request_t *r,
             do_write = 1;
         }
 
-        if (b->start == NULL) {
-            b->start = ngx_palloc(r->pool, u->conf->buffer_size);
-            if (b->start == NULL) {
-                ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
-                return;
-            }
+    }
 
-            b->pos = b->start;
-            b->last = b->start;
-            b->end = b->start + u->conf->buffer_size;
-            b->temporary = 1;
-            b->tag = u->output.tag;
+    if (b->start == NULL) {
+        b->start = ngx_palloc(r->pool, u->conf->buffer_size);
+        if (b->start == NULL) {
+            ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
+            return;
         }
+
+        b->pos = b->start;
+        b->last = b->start;
+        b->end = b->start + u->conf->buffer_size;
+        b->temporary = 1;
+        b->tag = u->output.tag;
     }
 
     for ( ;; ) {
diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h
index fd642c2d..b4fcbabc 100644
--- a/src/http/ngx_http_upstream.h
+++ b/src/http/ngx_http_upstream.h
@@ -397,6 +397,9 @@ struct ngx_http_upstream_s {
     unsigned                         request_body_sent:1;
     unsigned                         request_body_blocked:1;
     unsigned                         header_sent:1;
+#if (NGX_HTTP_PROXY_FORWARD)
+    unsigned                         forward:1;
+#endif
 };
 
 


From alexander.borisov at nginx.com  Tue Jul 14 11:51:01 2020
From: alexander.borisov at nginx.com (Alexander Borisov)
Date: Tue, 14 Jul 2020 11:51:01 +0000
Subject: [njs] Parser: fixed parsing return statement without semicolon.
Message-ID: <hg.022995046310.1594727461.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/022995046310
branches:  
changeset: 1464:022995046310
user:      Alexander Borisov <alexander.borisov at nginx.com>
date:      Tue Jul 14 14:49:46 2020 +0300
description:
Parser: fixed parsing return statement without semicolon.

The issue was introduced in 86f55a7dc4a4.

This closes #330 issue on GitHub.

diffstat:

 src/njs_parser.c         |  3 +--
 src/test/njs_unit_test.c |  3 +++
 2 files changed, 4 insertions(+), 2 deletions(-)

diffs (33 lines):

diff -r 5f2034557bc3 -r 022995046310 src/njs_parser.c
--- a/src/njs_parser.c	Thu Jul 09 13:57:10 2020 +0000
+++ b/src/njs_parser.c	Tue Jul 14 14:49:46 2020 +0300
@@ -5719,6 +5719,7 @@ njs_parser_return_statement(njs_parser_t
 
     switch (token->type) {
     case NJS_TOKEN_SEMICOLON:
+        njs_lexer_consume_token(parser->lexer, 1);
         break;
 
     case NJS_TOKEN_LINE_END:
@@ -5741,8 +5742,6 @@ njs_parser_return_statement(njs_parser_t
 
     parser->node = node;
 
-    njs_lexer_consume_token(parser->lexer, 1);
-
     return njs_parser_stack_pop(parser);
 }
 
diff -r 5f2034557bc3 -r 022995046310 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Thu Jul 09 13:57:10 2020 +0000
+++ b/src/test/njs_unit_test.c	Tue Jul 14 14:49:46 2020 +0300
@@ -2553,6 +2553,9 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("var x = 0, y = 2; x\n--\ny; [x,y]"),
       njs_str("0,1") },
 
+    { njs_str("function f() {return\n}"),
+      njs_str("undefined") },
+
     /* if. */
 
     { njs_str("if (0);"),

From xeioex at nginx.com  Tue Jul 14 13:16:52 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 14 Jul 2020 13:16:52 +0000
Subject: [njs] Fixed njs_number_to_int32() for big-endian platforms.
Message-ID: <hg.c1f74ea54f89.1594732612.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/c1f74ea54f89
branches:  
changeset: 1465:c1f74ea54f89
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 14 13:16:05 2020 +0000
description:
Fixed njs_number_to_int32() for big-endian platforms.

This is related to #326 issue in Github.

diffstat:

 auto/clang       |  17 -------------
 src/njs_number.h |  73 ++++++++++++++++++++++++++++---------------------------
 2 files changed, 37 insertions(+), 53 deletions(-)

diffs (135 lines):

diff -r 022995046310 -r c1f74ea54f89 auto/clang
--- a/auto/clang	Tue Jul 14 14:49:46 2020 +0300
+++ b/auto/clang	Tue Jul 14 13:16:05 2020 +0000
@@ -162,23 +162,6 @@ njs_feature_test="#include <sanitizer/ms
 . auto/feature
 
 
-njs_feature="NAN to uint conversion"
-njs_feature_name=NJS_NAN_TO_UINT_CONVERSION
-njs_feature_run=value
-njs_feature_incs=
-njs_feature_libs=-lm
-njs_feature_test="#include <math.h>
-                  #include <stdio.h>
-                  #include <stdint.h>
-
-                  int main(void) {
-                      int64_t  i64 = acosh(0);
-                      printf(\"%x\", (uint32_t) i64);
-                      return 0;
-                  }"
-. auto/feature
-
-
 njs_feature="_mm_setcsr()"
 njs_feature_name=NJS_HAVE_DENORMALS_CONTROL
 njs_feature_run=no
diff -r 022995046310 -r c1f74ea54f89 src/njs_number.h
--- a/src/njs_number.h	Tue Jul 14 14:49:46 2020 +0300
+++ b/src/njs_number.h	Tue Jul 14 13:16:05 2020 +0000
@@ -52,38 +52,6 @@ njs_key_is_integer_index(double num, con
 
 
 njs_inline int64_t
-njs_number_to_int64(double num)
-{
-#if (NJS_NAN_TO_UINT_CONVERSION != 0)
-    /*
-     * PPC32: NaN and Inf are converted to 0x8000000080000000
-     * and become non-zero after truncation.
-     */
-
-    if (isnan(num) || isinf(num)) {
-        return 0;
-    }
-#endif
-
-    /*
-     * ES5.1: integer must be modulo 2^32.
-     * 2^53 is the largest integer number which can be stored safely
-     * in the IEEE-754 format and numbers less than 2^53 can be just
-     * converted to int64_t eliding more expensive fmod() operation.
-     * Then the int64 integer is truncated to uint32_t.  The NaN is
-     * converted to 0x8000000000000000 and becomes 0 after truncation.
-     * fmod() of the Infinity returns NaN.
-     */
-
-    if (fabs(num) > 9007199254740992.0) {
-        return (int64_t) fmod(num, 4294967296.0);
-    }
-
-    return (int64_t) num;
-}
-
-
-njs_inline int64_t
 njs_number_to_integer(double num)
 {
     if (njs_slow_path(isinf(num))) {
@@ -97,28 +65,61 @@ njs_number_to_integer(double num)
         return 0;
     }
 
-    return njs_number_to_int64(num);
+    return trunc(num) + 0.0;
 }
 
 
 njs_inline int32_t
 njs_number_to_int32(double num)
 {
-    return (int32_t) njs_number_to_int64(num);
+    int32_t           r;
+    uint64_t          v;
+    njs_int_t         exp;
+    njs_diyfp_conv_t  conv;
+
+    conv.d = num;
+
+    exp = (conv.u64 & NJS_DBL_EXPONENT_MASK) >> NJS_DBL_SIGNIFICAND_SIZE;
+
+    if (njs_fast_path(exp < (NJS_DBL_EXPONENT_OFFSET + 30))) {
+        /* |num| < 2**31. */
+        return num;
+    }
+
+    if (exp < (NJS_DBL_EXPONENT_OFFSET + 30 + 53)) {
+        v = (conv.u64 & NJS_DBL_SIGNIFICAND_MASK) | NJS_DBL_HIDDEN_BIT;
+        v <<= (exp - NJS_DBL_EXPONENT_BIAS + 32);
+        r = v >> 32;
+
+        if (conv.u64 & NJS_DBL_SIGN_MASK) {
+            r = -r;
+        }
+
+        return r;
+    }
+
+    /*
+     * ES5.1: integer must be modulo 2^32.
+     * The distance between larger doubles
+     * (exp >= NJS_DBL_EXPONENT_OFFSET + 30 + 53) is a multiple of 2**32 => 0.
+     * This also handles NaN and Inf.
+     */
+
+    return 0;
 }
 
 
 njs_inline uint32_t
 njs_number_to_uint32(double num)
 {
-    return (uint32_t) njs_number_to_int64(num);
+    return (uint32_t) njs_number_to_int32(num);
 }
 
 
 njs_inline uint16_t
 njs_number_to_uint16(double num)
 {
-    return (uint16_t) njs_number_to_int64(num);
+    return (uint16_t) njs_number_to_int32(num);
 }
 
 

From xeioex at nginx.com  Tue Jul 14 13:16:54 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 14 Jul 2020 13:16:54 +0000
Subject: [njs] Fixed unit test on big-endian platforms.
Message-ID: <hg.fcb5e172abaf.1594732614.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/fcb5e172abaf
branches:  
changeset: 1466:fcb5e172abaf
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 14 13:16:06 2020 +0000
description:
Fixed unit test on big-endian platforms.

This closes #326 issue on Github.

diffstat:

 auto/endianness          |  36 ++++++++++++++++++++++++++++++++++++
 configure                |   1 +
 src/test/njs_unit_test.c |  41 ++++++++++++++++++++++++++++++++---------
 3 files changed, 69 insertions(+), 9 deletions(-)

diffs (150 lines):

diff -r c1f74ea54f89 -r fcb5e172abaf auto/endianness
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/auto/endianness	Tue Jul 14 13:16:06 2020 +0000
@@ -0,0 +1,36 @@
+
+# Copyright (C) Dmitry Volyntsev
+# Copyright (C) NGINX, Inc.
+
+njs_found=no
+NJS_BYTE_ORDER=little
+
+njs_feature="system byte ordering"
+njs_feature_name=NJS_BYTE_ORDER
+njs_feature_run=value
+njs_feature_incs=
+njs_feature_libs=-lm
+njs_feature_test="#include <stdio.h>
+                  #include <stdint.h>
+
+                  int main(void) {
+                      uint16_t x = 1; /* 0x0001 */
+                      printf(\"%s\n\", (*((uint8_t *) &x) == 0) ? \"big\" : \"little\");
+                      return 0;
+                  }"
+. auto/feature
+
+if [ $njs_found = no ]; then
+    echo
+    echo "$0: error: cannot detect system byte ordering"
+    echo
+    exit 1;
+fi
+
+if [ $NJS_BYTE_ORDER = big ]; then
+    njs_define=NJS_HAVE_BIG_ENDIAN . auto/define
+
+else
+    njs_define=NJS_HAVE_LITTLE_ENDIAN . auto/define
+
+fi
diff -r c1f74ea54f89 -r fcb5e172abaf configure
--- a/configure	Tue Jul 14 13:16:05 2020 +0000
+++ b/configure	Tue Jul 14 13:16:06 2020 +0000
@@ -48,6 +48,7 @@ NJS_LIBRT=
 . auto/options
 . auto/cc
 . auto/types
+. auto/endianness
 . auto/clang
 . auto/time
 . auto/memalign
diff -r c1f74ea54f89 -r fcb5e172abaf src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Tue Jul 14 13:16:05 2020 +0000
+++ b/src/test/njs_unit_test.c	Tue Jul 14 13:16:06 2020 +0000
@@ -14,6 +14,12 @@
 #define NJS_LARGE_ARRAY      _NJS_ARRAY(NJS_ARRAY_LARGE_OBJECT_LENGTH + 1)
 #define NJS_LARGE_ARRAY_LEN  "32769"
 
+#ifdef NJS_HAVE_LITTLE_ENDIAN
+#define njs_evar(little, big) (little)
+#else
+#define njs_evar(little, big) (big)
+#endif
+
 
 typedef struct {
     njs_str_t  script;
@@ -5411,17 +5417,22 @@ static njs_unit_test_t  njs_test[] =
     { njs_str(NJS_INT_TYPED_ARRAY_LIST
               ".map(v=>{var init = new Uint32Array([0xaabbccdd]);"
               "     try { return new v(init.buffer, 0, 2)} catch (e) {return e.name}})"),
-      njs_str("221,204,221,204,-35,-52,52445,43707,-13091,-21829,RangeError,RangeError") },
+      njs_str(njs_evar("221,204,221,204,-35,-52,52445,43707,-13091,-21829,RangeError,RangeError",
+                       "170,187,170,187,-86,-69,43707,52445,-21829,-13091,RangeError,RangeError")) },
 
     { njs_str(NJS_INT_TYPED_ARRAY_LIST
               ".map(v=>{var init = new Uint32Array([0xaabbccdd]);"
               "     try { return new v(init.buffer, 1, 2)} catch (e) {return e.name}})"),
-      njs_str("204,187,204,187,-52,-69,RangeError,RangeError,RangeError,RangeError") },
+      njs_str(njs_evar("204,187,204,187,-52,-69,RangeError,RangeError,RangeError,RangeError",
+                       "187,204,187,204,-69,-52,RangeError,RangeError,RangeError,RangeError")) },
 
     { njs_str(NJS_INT_TYPED_ARRAY_LIST
               ".map(v=>{var init = new Uint32Array([0xaabbccdd,0xdeadbeef]);"
               "     try { return new v(init.buffer, 0, 2)} catch (e) {return e.name}})"),
-      njs_str("221,204,221,204,-35,-52,52445,43707,-13091,-21829,2864434397,3735928559,-1430532899,-559038737") },
+      njs_str(njs_evar("221,204,221,204,-35,-52,52445,43707,-13091,-21829,"
+                       "2864434397,3735928559,-1430532899,-559038737",
+                       "170,187,170,187,-86,-69,43707,52445,-21829,-13091,"
+                       "2864434397,3735928559,-1430532899,-559038737")) },
 
     { njs_str(NJS_TYPED_ARRAY_LIST
               ".every(v=>{var buffer1 = new ArrayBuffer(8 * v.BYTES_PER_ELEMENT);"
@@ -5591,7 +5602,8 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("[1.0, -1234.0]"
               ".map(v=>{var a = new Float32Array(1); a[0] = v; var b = new Uint8Array(a.buffer);"
               "         return (b[0] << 24 | b[1] << 16| b[2] <<8 | b[3]).toString(16).padStart(8, '0');})"),
-      njs_str("0000803f,00409ac4") },
+      njs_str(njs_evar("0000803f,00409ac4",
+                       "3f800000,-3b65c000")) },
 
     { njs_str("var a = new ArrayBuffer(0); a.slice(0, 0).byteLength"),
       njs_str("0") },
@@ -5608,17 +5620,26 @@ static njs_unit_test_t  njs_test[] =
     { njs_str(NJS_TYPED_ARRAY_LIST
               ".map(v=>{var buffer = new ArrayBuffer(8); var view = new v(buffer);"
               "         view[0] = 511; return new Uint8Array(buffer.slice(0,4))})"),
-      njs_str("255,0,0,0,255,0,0,0,255,0,0,0,255,1,0,0,255,1,0,0,255,1,0,0,255,1,0,0,0,128,255,67,0,0,0,0") },
+      njs_str(njs_evar("255,0,0,0,255,0,0,0,255,0,0,0,255,1,0,0,255,1,"
+                       "0,0,255,1,0,0,255,1,0,0,0,128,255,67,0,0,0,0",
+                       "255,0,0,0,255,0,0,0,255,0,0,0,1,255,0,0,1,255,0,0,"
+                       "0,0,1,255,0,0,1,255,67,255,128,0,64,127,240,0")) },
 
     { njs_str(NJS_TYPED_ARRAY_LIST
               ".map(v=>{var buffer = new ArrayBuffer(8); var view = new v(buffer);"
               "         view[view.length - 1] = 511; return new Uint8Array(buffer.slice(4))})"),
-      njs_str("0,0,0,255,0,0,0,255,0,0,0,255,0,0,255,1,0,0,255,1,255,1,0,0,255,1,0,0,0,128,255,67,0,240,127,64") },
+      njs_str(njs_evar("0,0,0,255,0,0,0,255,0,0,0,255,0,0,255,1,0,0,255,1,"
+                       "255,1,0,0,255,1,0,0,0,128,255,67,0,240,127,64",
+                       "0,0,0,255,0,0,0,255,0,0,0,255,0,0,1,255,0,0,1,255,"
+                       "0,0,1,255,0,0,1,255,67,255,128,0,0,0,0,0")) },
 
     { njs_str(NJS_TYPED_ARRAY_LIST
               ".map(v=>{var buffer = new ArrayBuffer(8); var view = new v(buffer);"
               "         view[0] = 511; return new Uint8Array(buffer.slice(0,-4))})"),
-      njs_str("255,0,0,0,255,0,0,0,255,0,0,0,255,1,0,0,255,1,0,0,255,1,0,0,255,1,0,0,0,128,255,67,0,0,0,0") },
+      njs_str(njs_evar("255,0,0,0,255,0,0,0,255,0,0,0,255,1,0,0,255,1,0,0,"
+                       "255,1,0,0,255,1,0,0,0,128,255,67,0,0,0,0",
+                       "255,0,0,0,255,0,0,0,255,0,0,0,1,255,0,0,1,255,0,0,"
+                       "0,0,1,255,0,0,1,255,67,255,128,0,64,127,240,0")) },
 
     { njs_str("var a = new Uint8Array(10); var b = a.slice(1); b.length"),
       njs_str("9") },
@@ -5641,12 +5662,14 @@ static njs_unit_test_t  njs_test[] =
     { njs_str(NJS_INT_TYPED_ARRAY_LIST
               ".map(v=>{var init = new Uint8Array([1,2,3,4,5,6,7,8]); var view = new v(init.buffer);"
               "         return view.slice(0,2)})"),
-      njs_str("1,2,1,2,1,2,513,1027,513,1027,67305985,134678021,67305985,134678021") },
+      njs_str(njs_evar("1,2,1,2,1,2,513,1027,513,1027,67305985,134678021,67305985,134678021",
+                       "1,2,1,2,1,2,258,772,258,772,16909060,84281096,16909060,84281096")) },
 
     { njs_str(NJS_INT_TYPED_ARRAY_LIST
               ".map(v=>{var init = new Uint8Array([1,2,3,4,5,6,7,8]); var view = new v(init.buffer);"
               "         return view.slice(0,-2)})"),
-      njs_str("1,2,3,4,5,6,1,2,3,4,5,6,1,2,3,4,5,6,513,1027,513,1027,,") },
+      njs_str(njs_evar("1,2,3,4,5,6,1,2,3,4,5,6,1,2,3,4,5,6,513,1027,513,1027,,",
+                       "1,2,3,4,5,6,1,2,3,4,5,6,1,2,3,4,5,6,258,772,258,772,,")) },
 
     { njs_str("var other = new Uint8Array([0xff,0xff,0xff,0xff]);"
               NJS_TYPED_ARRAY_LIST

From mdounin at mdounin.ru  Tue Jul 14 14:27:03 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 14 Jul 2020 17:27:03 +0300
Subject: HTTP: support http forward proxying
In-Reply-To: <20200714034126.6ED25497AFD@mail.nginx.com>
References: <20200714034126.6ED25497AFD@mail.nginx.com>
Message-ID: <20200714142703.GA12747@mdounin.ru>

Hello!

On Tue, Jul 14, 2020 at 11:41:03AM +0800, ??? wrote:

> Hello, nginx,
> 
> This is my first try for adding the http forward proxying feature to nginx.
> 
> The http forward proxy protocol runs on plain http connection, which has big privacy issue.
> So web browsers like chrome and firefox has added support for forward proxy over https.
> 
> We use nginx as https server, and we also need the forward proxy feature over https on the same host.
> Nginx does not support forward proxying now. If we launch another security http proxy server, we
> have no choice but stop the nginx server, because both the proxy server and nginx will listen on the
> same 443 port.
> 
> So, the http forward proxying is need for nginx for us.
> 
> After investigate the nginx source, I found that almost all features needed for http forward proxying
> are there. If we implement this feature as a dedicate module, we have to duplicate too many code
> from the http_proxy_module and http_upstream_module. So I propose to make small modification to the
> exists code base.

[...]

> Here is the patch. Please make your comment. Thank you. 

Thank you for the patch.

Forward proxying is intentionally not supported by nginx.  In many 
places the code assumes trusted backends and either provides 
defaults which are not suitable for forward proxying (such as 
X-Accel-* headers being handled by default), or provides no 
mitigations for various possible malicious behaviour of upstream 
servers.

While using carefully configured nginx as a forward proxy might be 
possible in some cases, this is not something we are going to 
support.  Accordingly, no patches to support additional aspects of 
forward proxying will be accepted.  If you need a forward proxy, 
consider other products.

Thank you for understanding.

-- 
Maxim Dounin
http://mdounin.ru/

From xeioex at nginx.com  Tue Jul 14 17:22:18 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 14 Jul 2020 17:22:18 +0000
Subject: [njs] Parser: fixed regexp-literals parsing with '=' characters.
Message-ID: <hg.42d20205e054.1594747338.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/42d20205e054
branches:  
changeset: 1467:42d20205e054
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 14 17:20:29 2020 +0000
description:
Parser: fixed regexp-literals parsing with '=' characters.

Previously, njs lexer decoded '/=' as an assignment token.

This closes #329 issue on Github.

diffstat:

 src/njs_parser.c         |  5 +++++
 src/test/njs_unit_test.c |  6 ++++++
 2 files changed, 11 insertions(+), 0 deletions(-)

diffs (45 lines):

diff -r fcb5e172abaf -r 42d20205e054 src/njs_parser.c
--- a/src/njs_parser.c	Tue Jul 14 13:16:06 2020 +0000
+++ b/src/njs_parser.c	Tue Jul 14 17:20:29 2020 +0000
@@ -1115,6 +1115,7 @@ njs_parser_primary_expression_test(njs_p
 
     /* RegularExpressionLiteral */
     case NJS_TOKEN_DIVISION:
+    case NJS_TOKEN_DIVISION_ASSIGNMENT:
         node = njs_parser_node_new(parser, NJS_TOKEN_REGEXP);
         if (node == NULL) {
             return NJS_ERROR;
@@ -1201,6 +1202,10 @@ njs_parser_regexp_literal(njs_parser_t *
     value = &parser->node->u.value;
     lexer = parser->lexer;
 
+    if (token->type == NJS_TOKEN_DIVISION_ASSIGNMENT) {
+        lexer->start--;
+    }
+
     for (p = lexer->start; p < lexer->end; p++) {
 
         switch (*p) {
diff -r fcb5e172abaf -r 42d20205e054 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Tue Jul 14 13:16:06 2020 +0000
+++ b/src/test/njs_unit_test.c	Tue Jul 14 17:20:29 2020 +0000
@@ -7582,6 +7582,9 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("'abc'.replace(/f/, 'X')"),
       njs_str("abc") },
 
+    { njs_str("'AB=C==='.replace(/=*$/, '')"),
+      njs_str("AB=C") },
+
     { njs_str("('a'.repeat(33) + 'bb').replace(/bb/, 'CC').slice(31)"),
       njs_str("aaCC") },
 
@@ -7784,6 +7787,9 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("/]/"),
       njs_str("/\\]/") },
 
+    { njs_str("/=/"),
+      njs_str("/=/") },
+
     { njs_str("/["),
       njs_str("SyntaxError: Unterminated RegExp \"/[\" in 1") },
 

From xeioex at nginx.com  Tue Jul 14 17:22:20 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 14 Jul 2020 17:22:20 +0000
Subject: [njs] Style.
Message-ID: <hg.3dc7a53d9f00.1594747340.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/3dc7a53d9f00
branches:  
changeset: 1468:3dc7a53d9f00
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 14 17:21:53 2020 +0000
description:
Style.

diffstat:

 src/njs_parser.c      |  2 +-
 src/njs_typed_array.c |  2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diffs (24 lines):

diff -r 42d20205e054 -r 3dc7a53d9f00 src/njs_parser.c
--- a/src/njs_parser.c	Tue Jul 14 17:20:29 2020 +0000
+++ b/src/njs_parser.c	Tue Jul 14 17:21:53 2020 +0000
@@ -3140,7 +3140,7 @@ njs_parser_left_hand_side_expression_opt
 
 static njs_int_t
 njs_parser_expression_node(njs_parser_t *parser, njs_lexer_token_t *token,
-	njs_queue_link_t *current, njs_token_type_t type,
+    njs_queue_link_t *current, njs_token_type_t type,
     njs_vmcode_operation_t operation, njs_parser_state_func_t after)
 {
     njs_parser_node_t  *node;
diff -r 42d20205e054 -r 3dc7a53d9f00 src/njs_typed_array.c
--- a/src/njs_typed_array.c	Tue Jul 14 17:20:29 2020 +0000
+++ b/src/njs_typed_array.c	Tue Jul 14 17:21:53 2020 +0000
@@ -597,7 +597,7 @@ njs_typed_array_prototype_fill(njs_vm_t 
         for (i = start; i < end; i++) {
             buffer->u.f64[i + offset] = num;
         }
-	}
+    }
 
     return NJS_OK;
 }

From xeioex at nginx.com  Wed Jul 15 13:51:42 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Wed, 15 Jul 2020 13:51:42 +0000
Subject: [njs] Tests: improved declaring of sparse arrays in unit tests.
Message-ID: <hg.2ed052ecdc66.1594821102.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/2ed052ecdc66
branches:  
changeset: 1469:2ed052ecdc66
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Jul 14 18:22:03 2020 +0000
description:
Tests: improved declaring of sparse arrays in unit tests.

diffstat:

 src/test/njs_unit_test.c |  40 +++++++++++++++++++++-------------------
 1 files changed, 21 insertions(+), 19 deletions(-)

diffs (114 lines):

diff -r 3dc7a53d9f00 -r 2ed052ecdc66 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Tue Jul 14 17:21:53 2020 +0000
+++ b/src/test/njs_unit_test.c	Tue Jul 14 18:22:03 2020 +0000
@@ -10,9 +10,6 @@
 
 
 #define NJS_HAVE_LARGE_STACK (!NJS_HAVE_ADDRESS_SANITIZER && !NJS_HAVE_MEMORY_SANITIZER)
-#define _NJS_ARRAY(sz)       "Array(" njs_stringify(sz) ")"
-#define NJS_LARGE_ARRAY      _NJS_ARRAY(NJS_ARRAY_LARGE_OBJECT_LENGTH + 1)
-#define NJS_LARGE_ARRAY_LEN  "32769"
 
 #ifdef NJS_HAVE_LITTLE_ENDIAN
 #define njs_evar(little, big) (little)
@@ -21,6 +18,13 @@
 #endif
 
 
+#define njs_declare_sparse_array(nm, sz)                                      \
+    "var " nm " = Array(" njs_stringify(sz) "); "                             \
+    "Object.defineProperty(" nm ", '0',"                                      \
+    "{writable:true, enumerable:false, configurable:true});"                  \
+    "delete " nm "[0];"
+
+
 typedef struct {
     njs_str_t  script;
     njs_str_t  ret;
@@ -4021,15 +4025,15 @@ static njs_unit_test_t  njs_test[] =
               "x.concat().hasOwnProperty('1') === true"),
       njs_str("true") },
 
-    { njs_str("var a = " NJS_LARGE_ARRAY ";"
+    { njs_str(njs_declare_sparse_array("a", 64)
               "a[32] = 1; a = a.concat([1]);"
               "njs.dump([a[0], a[32],a.length])"),
-      njs_str("[undefined,1,32770]") },
-
-    { njs_str("var a = " NJS_LARGE_ARRAY ";"
+      njs_str("[undefined,1,65]") },
+
+    { njs_str(njs_declare_sparse_array("a", 64)
               "a[32] = 1; a = [1].concat(a);"
               "njs.dump([a[0], a[33],a.length])"),
-      njs_str("[1,1,32770]") },
+      njs_str("[1,1,65]") },
 
     { njs_str("var re = /abc/; re[Symbol.isConcatSpreadable] = true;"
               "re[0] = 1, re[1] = 2, re[2] = 3, re.length = 3;"
@@ -4064,8 +4068,8 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("Array.prototype.toString.call('abc')"),
       njs_str("[object String]") },
 
-    { njs_str("var a = " NJS_LARGE_ARRAY "; var s = a.toString();"
-              "[s.length]"),
+    { njs_str(njs_declare_sparse_array("a", 32769)
+              "var s = a.toString(); [s.length]"),
       njs_str("32768") },
 
     /* Empty array elements. */
@@ -4337,9 +4341,10 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("var o = { length: 3 }; Array.prototype.pop.call(o); o.length"),
       njs_str("2") },
 
-    { njs_str("var a = " NJS_LARGE_ARRAY "; a[a.length - 1] = 'z'; a[a.length -2] = 'y';"
+    { njs_str(njs_declare_sparse_array("a", 16)
+              "a[a.length - 1] = 'z'; a[a.length -2] = 'y';"
               "Array.prototype.pop.call(a); [a.length, a[a.length - 1]]"),
-      njs_str("32768,y") },
+      njs_str("15,y") },
 
     { njs_str("[0,1].slice()"),
       njs_str("0,1") },
@@ -6202,13 +6207,13 @@ static njs_unit_test_t  njs_test[] =
               "Array.prototype.join.call(a)"),
       njs_str("1,5,8,23") },
 
-    { njs_str("var a = " NJS_LARGE_ARRAY "; "
+    { njs_str(njs_declare_sparse_array("a", 1024)
               "a[100] = 1; a[512] = -1; a[5] = undefined;"
               "a.sort();"
               "a[0] == -1 && a[1] == 1 && a[2] == undefined"),
       njs_str("true") },
 
-    { njs_str("var a = " NJS_LARGE_ARRAY "; "
+    { njs_str(njs_declare_sparse_array("a", 1024)
               "a.fill(1, 256, 512); a.fill(undefined, 1000, 1010);"
               "Object.defineProperty(a, '256', {value: a[256], enumerable:false});"
               "a.sort();"
@@ -10964,9 +10969,6 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("var a = Array(Infinity)"),
       njs_str("RangeError: Invalid array length") },
 
-    { njs_str(NJS_LARGE_ARRAY ".length"),
-      njs_str(NJS_LARGE_ARRAY_LEN) },
-
     { njs_str("var a = Array(1111111111); a[1111111112] = 1; a.length"),
       njs_str("1111111113") },
 
@@ -15882,13 +15884,13 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("var a = [1]; a[2] = 'x'; JSON.stringify(a)"),
       njs_str("[1,null,\"x\"]") },
 
-    { njs_str("var a = " NJS_LARGE_ARRAY ";"
+    { njs_str(njs_declare_sparse_array("a", 32769)
               "a[32] = 'a'; a[64] = 'b';"
               "var s = JSON.stringify(a); "
               "[s.length,s.substring(162,163),s.match(/null/g).length]"),
       njs_str("163844,a,32767") },
 
-    { njs_str("var a = " NJS_LARGE_ARRAY ";"
+    { njs_str(njs_declare_sparse_array("a", 8)
               "a[2] = 'a'; a[4] = 'b'; a.length = 3;"
               "JSON.stringify(a)"),
       njs_str("[null,null,\"a\"]") },

From xeioex at nginx.com  Wed Jul 15 15:38:26 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Wed, 15 Jul 2020 15:38:26 +0000
Subject: [njs] Fixed detection of endianness introduced in fcb5e172abaf.
Message-ID: <hg.c39329b57a06.1594827506.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/c39329b57a06
branches:  
changeset: 1470:c39329b57a06
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Wed Jul 15 15:34:16 2020 +0000
description:
Fixed detection of endianness introduced in fcb5e172abaf.

Previously, NJS_HAVE_LITTLE_ENDIAN was declared unconditionally.

This correctly fixes #326 issue on Github.

diffstat:

 auto/endianness |  3 +--
 1 files changed, 1 insertions(+), 2 deletions(-)

diffs (20 lines):

diff -r 2ed052ecdc66 -r c39329b57a06 auto/endianness
--- a/auto/endianness	Tue Jul 14 18:22:03 2020 +0000
+++ b/auto/endianness	Wed Jul 15 15:34:16 2020 +0000
@@ -3,7 +3,6 @@
 # Copyright (C) NGINX, Inc.
 
 njs_found=no
-NJS_BYTE_ORDER=little
 
 njs_feature="system byte ordering"
 njs_feature_name=NJS_BYTE_ORDER
@@ -27,7 +26,7 @@ if [ $njs_found = no ]; then
     exit 1;
 fi
 
-if [ $NJS_BYTE_ORDER = big ]; then
+if [ $njs_feature_value = big ]; then
     njs_define=NJS_HAVE_BIG_ENDIAN . auto/define
 
 else

From alexander.borisov at nginx.com  Wed Jul 15 16:20:42 2020
From: alexander.borisov at nginx.com (Alexander Borisov)
Date: Wed, 15 Jul 2020 16:20:42 +0000
Subject: [njs] Introduced UTF-16 according to WHATWG encoding spec.
Message-ID: <hg.63106bd2e9bf.1594830042.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/63106bd2e9bf
branches:  
changeset: 1471:63106bd2e9bf
user:      Alexander Borisov <alexander.borisov at nginx.com>
date:      Wed Jul 15 19:19:18 2020 +0300
description:
Introduced UTF-16 according to WHATWG encoding spec.

diffstat:

 auto/make                    |    4 +-
 auto/sources                 |    3 +-
 src/njs_main.h               |    2 +
 src/njs_unicode.h            |   23 +++
 src/njs_utf16.c              |  116 +++++++++++++++
 src/njs_utf16.h              |   25 +++
 src/test/unicode_unit_test.c |  312 +++++++++++++++++++++++++++++++++++++++++++
 src/test/utf8_unit_test.c    |  202 ---------------------------
 8 files changed, 482 insertions(+), 205 deletions(-)

diffs (749 lines):

diff -r c39329b57a06 -r 63106bd2e9bf auto/make
--- a/auto/make	Wed Jul 15 15:34:16 2020 +0000
+++ b/auto/make	Wed Jul 15 19:19:18 2020 +0300
@@ -241,12 +241,12 @@ lib_test: $NJS_BUILD_DIR/njs_auto_config
 	$NJS_BUILD_DIR/random_unit_test \\
 	$NJS_BUILD_DIR/rbtree_unit_test \\
 	$NJS_BUILD_DIR/lvlhsh_unit_test \\
-	$NJS_BUILD_DIR/utf8_unit_test
+	$NJS_BUILD_DIR/unicode_unit_test
 
 	$NJS_BUILD_DIR/random_unit_test
 	$NJS_BUILD_DIR/rbtree_unit_test
 	$NJS_BUILD_DIR/lvlhsh_unit_test
-	$NJS_BUILD_DIR/utf8_unit_test
+	$NJS_BUILD_DIR/unicode_unit_test
 
 unit_test: $NJS_BUILD_DIR/njs_auto_config.h \\
 	$NJS_BUILD_DIR/njs_unit_test
diff -r c39329b57a06 -r 63106bd2e9bf auto/sources
--- a/auto/sources	Wed Jul 15 15:34:16 2020 +0000
+++ b/auto/sources	Wed Jul 15 19:19:18 2020 +0300
@@ -6,6 +6,7 @@ NJS_LIB_SRCS=" \
    src/njs_murmur_hash.c \
    src/njs_djb_hash.c \
    src/njs_utf8.c \
+   src/njs_utf16.c \
    src/njs_arr.c \
    src/njs_rbtree.c \
    src/njs_lvlhsh.c \
@@ -60,7 +61,7 @@ NJS_LIB_TEST_SRCS=" \
    src/test/lvlhsh_unit_test.c \
    src/test/random_unit_test.c \
    src/test/rbtree_unit_test.c \
-   src/test/utf8_unit_test.c \
+   src/test/unicode_unit_test.c \
 "
 
 NJS_TEST_SRCS=" \
diff -r c39329b57a06 -r 63106bd2e9bf src/njs_main.h
--- a/src/njs_main.h	Wed Jul 15 15:34:16 2020 +0000
+++ b/src/njs_main.h	Wed Jul 15 19:19:18 2020 +0300
@@ -14,7 +14,9 @@
 #include <njs_types.h>
 #include <njs_clang.h>
 #include <njs_str.h>
+#include <njs_unicode.h>
 #include <njs_utf8.h>
+#include <njs_utf16.h>
 #include <njs_diyfp.h>
 #include <njs_dtoa.h>
 #include <njs_dtoa_fixed.h>
diff -r c39329b57a06 -r 63106bd2e9bf src/njs_unicode.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/njs_unicode.h	Wed Jul 15 19:19:18 2020 +0300
@@ -0,0 +1,23 @@
+
+/*
+ * Copyright (C) Alexander Borisov
+ * Copyright (C) NGINX, Inc.
+ */
+
+#ifndef _NJS_UNICODE_H_INCLUDED_
+#define _NJS_UNICODE_H_INCLUDED_
+
+
+enum {
+    NJS_UNICODE_MAX_CODEPOINT = 0x10FFFF,
+    NJS_UNICODE_ERROR         = 0x1FFFFF,
+    NJS_UNICODE_CONTINUE      = 0x2FFFFF
+};
+
+typedef struct {
+    uint32_t  codepoint;
+    u_char    upper;
+} njs_unicode_decode_t;
+
+
+#endif /* _NJS_UNICODE_H_INCLUDED_ */
diff -r c39329b57a06 -r 63106bd2e9bf src/njs_utf16.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/njs_utf16.c	Wed Jul 15 19:19:18 2020 +0300
@@ -0,0 +1,116 @@
+
+/*
+ * Copyright (C) Alexander Borisov
+ * Copyright (C) NGINX, Inc.
+ */
+
+
+#include <njs_main.h>
+
+
+njs_inline void
+njs_utf16_encode_write(uint32_t cp, u_char **start)
+{
+#ifdef NJS_HAVE_BIG_ENDIAN
+        *(*start)++ = cp >> 8;
+        *(*start)++ = cp & 0x00FF;
+#else
+        *(*start)++ = cp & 0x00FF;
+        *(*start)++ = cp >> 8;
+#endif
+}
+
+
+ssize_t
+njs_utf16_encode(uint32_t cp, u_char **start, const u_char *end)
+{
+    if ((*start + 2) > end) {
+        return NJS_ERROR;
+    }
+
+    if (cp < 0x10000) {
+        njs_utf16_encode_write(cp, start);
+
+        return 2;
+    }
+
+    if ((*start + 4) > end) {
+        return NJS_ERROR;
+    }
+
+    cp -= 0x10000;
+
+    njs_utf16_encode_write((0xD800 | (cp >> 0x0A)), start);
+    njs_utf16_encode_write((0xDC00 | (cp & 0x03FF)), start);
+
+    return 4;
+}
+
+
+uint32_t
+njs_utf16_decode(njs_unicode_decode_t *ctx, const u_char **start,
+    const u_char *end)
+{
+    uint32_t  unit;
+    unsigned  lead;
+
+    if (ctx->upper != 0x00) {
+        lead = ctx->upper - 0x01;
+        ctx->upper = 0x00;
+
+        goto lead_state;
+    }
+
+pair_state:
+
+    lead = *(*start)++;
+
+    if (*start >= end) {
+        ctx->upper = lead + 0x01;
+        return NJS_UNICODE_CONTINUE;
+    }
+
+lead_state:
+
+#ifdef NJS_HAVE_BIG_ENDIAN
+        unit = (lead << 8) + *(*start)++;
+#else
+        unit = (*(*start)++ << 8) + lead;
+#endif
+
+    if (ctx->codepoint != 0x00) {
+        if ((unsigned) (unit - 0xDC00) <= (0xDFFF - 0xDC00)) {
+            unit = 0x10000 + ((ctx->codepoint - 0xD800) << 10)
+                   + (unit - 0xDC00);
+
+            ctx->codepoint = 0x00;
+
+            return unit;
+        }
+
+        (*start)--;
+
+        ctx->upper = lead + 0x01;
+        ctx->codepoint = 0x00;
+
+        return NJS_UNICODE_ERROR;
+    }
+
+    /* Surrogate pair. */
+
+    if ((unsigned) (unit - 0xD800) <= (0xDFFF - 0xD800)) {
+        if ((unsigned) (unit - 0xDC00) <= (0xDFFF - 0xDC00)) {
+            return NJS_UNICODE_ERROR;
+        }
+
+        ctx->codepoint = unit;
+
+        if (*start >= end) {
+            return NJS_UNICODE_CONTINUE;
+        }
+
+        goto pair_state;
+    }
+
+    return unit;
+}
diff -r c39329b57a06 -r 63106bd2e9bf src/njs_utf16.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/njs_utf16.h	Wed Jul 15 19:19:18 2020 +0300
@@ -0,0 +1,25 @@
+
+/*
+ * Copyright (C) Alexander Borisov
+ * Copyright (C) NGINX, Inc.
+ */
+
+#ifndef _NJS_UTF16_H_INCLUDED_
+#define _NJS_UTF16_H_INCLUDED_
+
+
+NJS_EXPORT ssize_t njs_utf16_encode(uint32_t cp, u_char **start,
+    const u_char *end);
+NJS_EXPORT uint32_t njs_utf16_decode(njs_unicode_decode_t *ctx,
+    const u_char **start, const u_char *end);
+
+
+njs_inline void
+njs_utf16_decode_init(njs_unicode_decode_t *ctx)
+{
+    ctx->upper = 0x00;
+    ctx->codepoint = 0x00;
+}
+
+
+#endif /* _NJS_UTF16_H_INCLUDED_ */
diff -r c39329b57a06 -r 63106bd2e9bf src/test/unicode_unit_test.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/test/unicode_unit_test.c	Wed Jul 15 19:19:18 2020 +0300
@@ -0,0 +1,312 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) NGINX, Inc.
+ */
+
+
+#include <njs_main.h>
+
+
+#define NJS_UTF8_START_TEST  0xC2
+
+
+static u_char  invalid[] = {
+
+    /* Invalid first byte less than 0xC2. */
+    1, 0x80, 0x00, 0x00, 0x00,
+    1, 0xC0, 0x00, 0x00, 0x00,
+    2, 0xC0, 0x00, 0x00, 0x00,
+    3, 0xC0, 0x00, 0x00, 0x00,
+    4, 0xC0, 0x00, 0x00, 0x00,
+
+    /* Invalid 0x0x110000 value. */
+    4, 0xF4, 0x90, 0x80, 0x80,
+
+    /* Incomplete length. */
+    2, 0xE0, 0xAF, 0xB5, 0x00,
+
+    /* Overlong values. */
+    2, 0xC0, 0x80, 0x00, 0x00,
+    2, 0xC1, 0xB3, 0x00, 0x00,
+    3, 0xE0, 0x80, 0x80, 0x00,
+    3, 0xE0, 0x81, 0xB3, 0x00,
+    3, 0xE0, 0x90, 0x9A, 0x00,
+    4, 0xF0, 0x80, 0x8A, 0x80,
+    4, 0xF0, 0x80, 0x81, 0xB3,
+    4, 0xF0, 0x80, 0xAF, 0xB5,
+};
+
+
+static njs_int_t
+utf8_overlong(u_char *overlong, size_t len)
+{
+    u_char        *p, utf8[4];
+    size_t        size;
+    uint32_t      u, d;
+    njs_uint_t    i;
+    const u_char  *pp;
+
+    pp = overlong;
+
+    d = njs_utf8_decode(&pp, overlong + len);
+
+    len = pp - overlong;
+
+    if (d != 0xFFFFFFFF) {
+        p = njs_utf8_encode(utf8, d);
+
+        size = (p != NULL) ? p - utf8 : 0;
+
+        if (len != size || memcmp(overlong, utf8, size) != 0) {
+
+            u = 0;
+            for (i = 0; i < len; i++) {
+                u = (u << 8) + overlong[i];
+            }
+
+            njs_printf("njs_utf8_decode(%05uXD, %uz) failed: %05uXD, %uz\n",
+                       u, len, d, size);
+
+            return NJS_ERROR;
+        }
+    }
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+utf8_unit_test(njs_uint_t start)
+{
+    u_char        *p, utf8[4];
+    size_t        len;
+    int32_t       n;
+    uint32_t      u, d;
+    njs_uint_t    i, k, l, m;
+    const u_char  *pp;
+
+    njs_printf("utf8 test started\n");
+
+    /* Test valid UTF-8. */
+
+    for (u = 0; u < 0x110000; u++) {
+
+        p = njs_utf8_encode(utf8, u);
+
+        if (p == NULL) {
+            njs_printf("njs_utf8_encode(%05uXD) failed\n", u);
+            return NJS_ERROR;
+        }
+
+        pp = utf8;
+
+        d = njs_utf8_decode(&pp, p);
+
+        if (u != d) {
+            njs_printf("njs_utf8_decode(%05uXD) failed: %05uxD\n", u, d);
+            return NJS_ERROR;
+        }
+    }
+
+    /* Test some invalid UTF-8. */
+
+    for (i = 0; i < sizeof(invalid); i += 5) {
+
+        len = invalid[i];
+        utf8[0] = invalid[i + 1];
+        utf8[1] = invalid[i + 2];
+        utf8[2] = invalid[i + 3];
+        utf8[3] = invalid[i + 4];
+
+        pp = utf8;
+
+        d = njs_utf8_decode(&pp, utf8 + len);
+
+        if (d != 0xFFFFFFFF) {
+
+            u = 0;
+            for (i = 0; i < len; i++) {
+                u = (u << 8) + utf8[i];
+            }
+
+            njs_printf("njs_utf8_decode(%05uXD, %uz) failed: %05uXD\n",
+                       u, len, d);
+            return NJS_ERROR;
+        }
+    }
+
+    /* Test all overlong UTF-8. */
+
+    for (i = start; i < 256; i++) {
+        utf8[0] = i;
+
+        if (utf8_overlong(utf8, 1) != NJS_OK) {
+            return NJS_ERROR;
+        }
+
+        for (k = 0; k < 256; k++) {
+            utf8[1] = k;
+
+            if (utf8_overlong(utf8, 2) != NJS_OK) {
+                return NJS_ERROR;
+            }
+
+            for (l = 0; l < 256; l++) {
+                utf8[2] = l;
+
+                if (utf8_overlong(utf8, 3) != NJS_OK) {
+                    return NJS_ERROR;
+                }
+
+                for (m = 0; m < 256; m++) {
+                    utf8[3] = m;
+
+                    if (utf8_overlong(utf8, 4) != NJS_OK) {
+                        return NJS_ERROR;
+                    }
+                }
+            }
+        }
+    }
+
+    n = njs_utf8_casecmp((u_char *) "ABC ??? ???",
+                         (u_char *) "abc ??? ???",
+                         njs_length("ABC ??? ???"),
+                         njs_length("abc ??? ???"));
+
+    if (n != 0) {
+        njs_printf("njs_utf8_casecmp() failed\n");
+        return NJS_ERROR;
+    }
+
+    njs_printf("utf8 test passed\n");
+    return NJS_OK;
+}
+
+
+static njs_int_t
+utf16_unit_test()
+{
+    int8_t                length, length_to;
+    u_char                *start, *end, *end_to;
+    uint32_t              cp, i;
+    njs_unicode_decode_t  ctx;
+    u_char                buf[8], to[4];
+
+    njs_printf("utf16 test started\n");
+
+    end = buf + sizeof(buf);
+    end_to = to + sizeof(to);
+
+    for (i = 0; i <= NJS_UNICODE_MAX_CODEPOINT; i++) {
+
+        /* Skip surrogate pair. */
+
+        if (i >= 0xD800 && i <= 0xDFFF) {
+            continue;
+        }
+
+        start = buf;
+
+        length = njs_utf16_encode(i, &start, end);
+        if (length < NJS_OK) {
+            njs_printf("utf16 test encode failed\n");
+            return NJS_ERROR;
+        }
+
+        njs_utf16_decode_init(&ctx);
+
+        start = buf;
+
+        cp = njs_utf16_decode(&ctx, (const u_char **) &start, start + length);
+        if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+            njs_printf("utf16 test decode failed\n");
+            return NJS_ERROR;
+        }
+
+        if (cp != i) {
+            njs_printf("utf16 test decode code point does not match\n");
+            return NJS_ERROR;
+        }
+
+        start = to;
+
+        length_to = njs_utf16_encode(cp, &start, end_to);
+        if (length_to < NJS_OK) {
+            njs_printf("utf16 test encode failed\n");
+            return NJS_ERROR;
+        }
+
+        if (length_to != length || njs_strncmp(buf, to, length) != 0) {
+            njs_printf("utf16 test decode-encode failed\n");
+            return NJS_ERROR;
+        }
+    }
+
+    /* Surrogate pair. */
+
+    for (i = 0xD800; i <= 0xDFFF; i++) {
+        start = buf;
+
+        length = njs_utf16_encode(i, &start, end);
+        if (length < NJS_OK) {
+            njs_printf("utf16 test surrogate pair encode lead failed\n");
+            return NJS_ERROR;
+        }
+
+        length_to = njs_utf16_encode(i - 0xD800 + 0xDC00, &start, end);
+        if (length_to < NJS_OK) {
+            njs_printf("utf16 test surrogate pair encode failed\n");
+            return NJS_ERROR;
+        }
+
+        njs_utf16_decode_init(&ctx);
+
+        start = buf;
+
+        cp = njs_utf16_decode(&ctx, (const u_char **) &start,
+                              start + length + length_to);
+        if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+            if (i < 0xDC00) {
+                njs_printf("utf16 test surrogate pair decode failed\n");
+                return NJS_ERROR;
+            }
+        }
+    }
+
+    njs_printf("utf16 test passed\n");
+
+    return NJS_OK;
+}
+
+
+int
+main(int argc, char **argv)
+{
+    njs_int_t   ret;
+    njs_uint_t  start;
+
+    njs_printf("unicode unit test started\n");
+
+    if (argc > 1 && argv[1][0] == 'a') {
+        start = NJS_UTF8_START_TEST;
+
+    } else {
+        start = 256;
+    }
+
+    ret = utf8_unit_test(start);
+    if (ret != NJS_OK) {
+        return ret;
+    }
+
+    ret = utf16_unit_test();
+    if (ret != NJS_OK) {
+        return ret;
+    }
+
+    njs_printf("unicode unit test passed\n");
+
+    return 0;
+}
diff -r c39329b57a06 -r 63106bd2e9bf src/test/utf8_unit_test.c
--- a/src/test/utf8_unit_test.c	Wed Jul 15 15:34:16 2020 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,202 +0,0 @@
-
-/*
- * Copyright (C) Igor Sysoev
- * Copyright (C) NGINX, Inc.
- */
-
-
-#include <njs_main.h>
-
-
-#define NJS_UTF8_START_TEST  0xC2
-//#define NJS_UTF8_START_TEST  0
-
-
-static u_char  invalid[] = {
-
-    /* Invalid first byte less than 0xC2. */
-    1, 0x80, 0x00, 0x00, 0x00,
-    1, 0xC0, 0x00, 0x00, 0x00,
-    2, 0xC0, 0x00, 0x00, 0x00,
-    3, 0xC0, 0x00, 0x00, 0x00,
-    4, 0xC0, 0x00, 0x00, 0x00,
-
-    /* Invalid 0x0x110000 value. */
-    4, 0xF4, 0x90, 0x80, 0x80,
-
-    /* Incomplete length. */
-    2, 0xE0, 0xAF, 0xB5, 0x00,
-
-    /* Overlong values. */
-    2, 0xC0, 0x80, 0x00, 0x00,
-    2, 0xC1, 0xB3, 0x00, 0x00,
-    3, 0xE0, 0x80, 0x80, 0x00,
-    3, 0xE0, 0x81, 0xB3, 0x00,
-    3, 0xE0, 0x90, 0x9A, 0x00,
-    4, 0xF0, 0x80, 0x8A, 0x80,
-    4, 0xF0, 0x80, 0x81, 0xB3,
-    4, 0xF0, 0x80, 0xAF, 0xB5,
-};
-
-
-static njs_int_t
-utf8_overlong(u_char *overlong, size_t len)
-{
-    u_char        *p, utf8[4];
-    size_t        size;
-    uint32_t      u, d;
-    njs_uint_t    i;
-    const u_char  *pp;
-
-    pp = overlong;
-
-    d = njs_utf8_decode(&pp, overlong + len);
-
-    len = pp - overlong;
-
-    if (d != 0xFFFFFFFF) {
-        p = njs_utf8_encode(utf8, d);
-
-        size = (p != NULL) ? p - utf8 : 0;
-
-        if (len != size || memcmp(overlong, utf8, size) != 0) {
-
-            u = 0;
-            for (i = 0; i < len; i++) {
-                u = (u << 8) + overlong[i];
-            }
-
-            njs_printf("njs_utf8_decode(%05uXD, %uz) failed: %05uXD, %uz\n",
-                       u, len, d, size);
-
-            return NJS_ERROR;
-        }
-    }
-
-    return NJS_OK;
-}
-
-
-static njs_int_t
-utf8_unit_test(njs_uint_t start)
-{
-    u_char        *p, utf8[4];
-    size_t        len;
-    int32_t       n;
-    uint32_t      u, d;
-    njs_uint_t    i, k, l, m;
-    const u_char  *pp;
-
-    njs_printf("utf8 unit test started\n");
-
-    /* Test valid UTF-8. */
-
-    for (u = 0; u < 0x110000; u++) {
-
-        p = njs_utf8_encode(utf8, u);
-
-        if (p == NULL) {
-            njs_printf("njs_utf8_encode(%05uXD) failed\n", u);
-            return NJS_ERROR;
-        }
-
-        pp = utf8;
-
-        d = njs_utf8_decode(&pp, p);
-
-        if (u != d) {
-            njs_printf("njs_utf8_decode(%05uXD) failed: %05uxD\n", u, d);
-            return NJS_ERROR;
-        }
-    }
-
-    /* Test some invalid UTF-8. */
-
-    for (i = 0; i < sizeof(invalid); i += 5) {
-
-        len = invalid[i];
-        utf8[0] = invalid[i + 1];
-        utf8[1] = invalid[i + 2];
-        utf8[2] = invalid[i + 3];
-        utf8[3] = invalid[i + 4];
-
-        pp = utf8;
-
-        d = njs_utf8_decode(&pp, utf8 + len);
-
-        if (d != 0xFFFFFFFF) {
-
-            u = 0;
-            for (i = 0; i < len; i++) {
-                u = (u << 8) + utf8[i];
-            }
-
-            njs_printf("njs_utf8_decode(%05uXD, %uz) failed: %05uXD\n",
-                       u, len, d);
-            return NJS_ERROR;
-        }
-    }
-
-    /* Test all overlong UTF-8. */
-
-    for (i = start; i < 256; i++) {
-        utf8[0] = i;
-
-        if (utf8_overlong(utf8, 1) != NJS_OK) {
-            return NJS_ERROR;
-        }
-
-        for (k = 0; k < 256; k++) {
-            utf8[1] = k;
-
-            if (utf8_overlong(utf8, 2) != NJS_OK) {
-                return NJS_ERROR;
-            }
-
-            for (l = 0; l < 256; l++) {
-                utf8[2] = l;
-
-                if (utf8_overlong(utf8, 3) != NJS_OK) {
-                    return NJS_ERROR;
-                }
-
-                for (m = 0; m < 256; m++) {
-                    utf8[3] = m;
-
-                    if (utf8_overlong(utf8, 4) != NJS_OK) {
-                        return NJS_ERROR;
-                    }
-                }
-            }
-        }
-    }
-
-    n = njs_utf8_casecmp((u_char *) "ABC ??? ???",
-                         (u_char *) "abc ??? ???",
-                         njs_length("ABC ??? ???"),
-                         njs_length("abc ??? ???"));
-
-    if (n != 0) {
-        njs_printf("njs_utf8_casecmp() failed\n");
-        return NJS_ERROR;
-    }
-
-    njs_printf("utf8 unit test passed\n");
-    return NJS_OK;
-}
-
-
-int
-main(int argc, char **argv)
-{
-    njs_uint_t  start;
-
-    if (argc > 1 && argv[1][0] == 'a') {
-        start = NJS_UTF8_START_TEST;
-
-    } else {
-        start = 256;
-    }
-
-    return utf8_unit_test(start);
-}

From alexander.borisov at nginx.com  Wed Jul 15 16:20:44 2020
From: alexander.borisov at nginx.com (Alexander Borisov)
Date: Wed, 15 Jul 2020 16:20:44 +0000
Subject: [njs] Introduced UTF-8 decoder according to WHATWG encoding spec.
Message-ID: <hg.855edd76bdb6.1594830044.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/855edd76bdb6
branches:  
changeset: 1472:855edd76bdb6
user:      Alexander Borisov <alexander.borisov at nginx.com>
date:      Wed Jul 15 19:19:19 2020 +0300
description:
Introduced UTF-8 decoder according to WHATWG encoding spec.

diffstat:

 src/njs_json.c               |    8 +-
 src/njs_parser.c             |   73 ++++++---
 src/njs_string.c             |  293 +++++++++++++++++++++----------------
 src/njs_unicode.h            |    4 +
 src/njs_utf8.c               |  333 ++++++++++++++++++++----------------------
 src/njs_utf8.h               |   40 ++--
 src/test/njs_unit_test.c     |   44 ++++-
 src/test/unicode_unit_test.c |   53 ++++--
 8 files changed, 466 insertions(+), 382 deletions(-)

diffs (truncated from 1394 to 1000 lines):

diff -r 63106bd2e9bf -r 855edd76bdb6 src/njs_json.c
--- a/src/njs_json.c	Wed Jul 15 19:19:18 2020 +0300
+++ b/src/njs_json.c	Wed Jul 15 19:19:19 2020 +0300
@@ -728,7 +728,7 @@ njs_json_parse_string(njs_json_parse_ctx
             if (njs_surrogate_any(utf)) {
 
                 if (utf > 0xdbff || p[0] != '\\' || p[1] != 'u') {
-                    s = njs_utf8_encode(s, NJS_UTF8_REPLACEMENT);
+                    s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT);
                     continue;
                 }
 
@@ -741,12 +741,12 @@ njs_json_parse_string(njs_json_parse_ctx
                     utf = njs_string_surrogate_pair(utf, utf_low);
 
                 } else if (njs_surrogate_leading(utf_low)) {
-                    utf = NJS_UTF8_REPLACEMENT;
-                    s = njs_utf8_encode(s, NJS_UTF8_REPLACEMENT);
+                    utf = NJS_UNICODE_REPLACEMENT;
+                    s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT);
 
                 } else {
                     utf = utf_low;
-                    s = njs_utf8_encode(s, NJS_UTF8_REPLACEMENT);
+                    s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT);
                 }
             }
 
diff -r 63106bd2e9bf -r 855edd76bdb6 src/njs_parser.c
--- a/src/njs_parser.c	Wed Jul 15 19:19:18 2020 +0300
+++ b/src/njs_parser.c	Wed Jul 15 19:19:19 2020 +0300
@@ -7896,11 +7896,12 @@ njs_int_t
 njs_parser_string_create(njs_vm_t *vm, njs_lexer_token_t *token,
     njs_value_t *value)
 {
-    u_char        *dst;
-    ssize_t       size, length;
-    uint32_t      cp;
-    njs_str_t     *src;
-    const u_char  *p, *end;
+    u_char                *dst;
+    ssize_t               size, length;
+    uint32_t              cp;
+    njs_str_t             *src;
+    const u_char          *p, *end;
+    njs_unicode_decode_t  ctx;
 
     src = &token->text;
 
@@ -7914,10 +7915,17 @@ njs_parser_string_create(njs_vm_t *vm, n
     p = src->start;
     end = src->start + src->length;
 
+    njs_utf8_decode_init(&ctx);
+
     while (p < end) {
-        cp = njs_utf8_safe_decode(&p, end);
-
-        dst = njs_utf8_encode(dst, cp);
+        cp = njs_utf8_decode(&ctx, &p, end);
+
+        if (cp <= NJS_UNICODE_MAX_CODEPOINT) {
+            dst = njs_utf8_encode(dst, cp);
+
+        } else {
+            dst = njs_utf8_encode(dst, NJS_UNICODE_REPLACEMENT);
+        }
     }
 
     if (length > NJS_STRING_MAP_STRIDE && size != length) {
@@ -7932,12 +7940,13 @@ static njs_token_type_t
 njs_parser_escape_string_create(njs_parser_t *parser, njs_lexer_token_t *token,
     njs_value_t *value)
 {
-    u_char        c, *start, *dst;
-    size_t        size, length, hex_length;
-    uint64_t      cp, cp_pair;
-    njs_int_t     ret;
-    njs_str_t     *string;
-    const u_char  *src, *end, *hex_end;
+    u_char                c, *start, *dst;
+    size_t                size, length, hex_length;
+    uint64_t              cp, cp_pair;
+    njs_int_t             ret;
+    njs_str_t             *string;
+    const u_char          *src, *end, *hex_end;
+    njs_unicode_decode_t  ctx;
 
     ret = njs_parser_escape_string_calc_length(parser, token, &size, &length);
     if (njs_slow_path(ret != NJS_OK)) {
@@ -8053,7 +8062,13 @@ njs_parser_escape_string_create(njs_pars
 
         src--;
 
-        cp = njs_utf8_safe_decode2(&src, end);
+        njs_utf8_decode_init(&ctx);
+
+        cp = njs_utf8_decode(&ctx, &src, end);
+        if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+            cp = NJS_UNICODE_REPLACEMENT;
+        }
+
         dst = njs_utf8_encode(dst, cp);
 
         continue;
@@ -8076,12 +8091,12 @@ njs_parser_escape_string_create(njs_pars
                 cp = njs_string_surrogate_pair(cp_pair, cp);
 
             } else if (njs_slow_path(njs_surrogate_leading(cp))) {
-                cp = NJS_UTF8_REPLACEMENT;
+                cp = NJS_UNICODE_REPLACEMENT;
 
                 dst = njs_utf8_encode(dst, (uint32_t) cp);
 
             } else {
-                dst = njs_utf8_encode(dst, NJS_UTF8_REPLACEMENT);
+                dst = njs_utf8_encode(dst, NJS_UNICODE_REPLACEMENT);
             }
 
             cp_pair = 0;
@@ -8092,7 +8107,7 @@ njs_parser_escape_string_create(njs_pars
                 continue;
             }
 
-            cp = NJS_UTF8_REPLACEMENT;
+            cp = NJS_UNICODE_REPLACEMENT;
         }
 
         dst = njs_utf8_encode(dst, (uint32_t) cp);
@@ -8116,10 +8131,11 @@ static njs_int_t
 njs_parser_escape_string_calc_length(njs_parser_t *parser,
     njs_lexer_token_t *token, size_t *out_size, size_t *out_length)
 {
-    size_t        size, length, hex_length;
-    uint64_t      cp, cp_pair;
-    njs_str_t     *string;
-    const u_char  *ptr, *src, *end, *hex_end;
+    size_t                size, length, hex_length;
+    uint64_t              cp, cp_pair;
+    njs_str_t             *string;
+    const u_char          *ptr, *src, *end, *hex_end;
+    njs_unicode_decode_t  ctx;
 
     size = 0;
     length = 0;
@@ -8173,7 +8189,12 @@ njs_parser_escape_string_calc_length(njs
         }
 
         if (*src >= 0x80) {
-            cp = njs_utf8_safe_decode2(&src, end);
+            njs_utf8_decode_init(&ctx);
+
+            cp = njs_utf8_decode(&ctx, &src, end);
+            if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+                cp = NJS_UNICODE_REPLACEMENT;
+            }
 
             size += njs_utf8_size(cp);
             length++;
@@ -8220,13 +8241,13 @@ njs_parser_escape_string_calc_length(njs
                 cp = njs_string_surrogate_pair(cp_pair, cp);
 
             } else if (njs_slow_path(njs_surrogate_leading(cp))) {
-                cp = NJS_UTF8_REPLACEMENT;
+                cp = NJS_UNICODE_REPLACEMENT;
 
                 size += njs_utf8_size(cp);
                 length++;
 
             } else {
-                size += njs_utf8_size(NJS_UTF8_REPLACEMENT);
+                size += njs_utf8_size(NJS_UNICODE_REPLACEMENT);
                 length++;
             }
 
@@ -8238,7 +8259,7 @@ njs_parser_escape_string_calc_length(njs
                 continue;
             }
 
-            cp = NJS_UTF8_REPLACEMENT;
+            cp = NJS_UNICODE_REPLACEMENT;
         }
 
         size += njs_utf8_size(cp);
diff -r 63106bd2e9bf -r 855edd76bdb6 src/njs_string.c
--- a/src/njs_string.c	Wed Jul 15 19:19:18 2020 +0300
+++ b/src/njs_string.c	Wed Jul 15 19:19:19 2020 +0300
@@ -20,10 +20,8 @@ static njs_int_t njs_string_slice_prop(n
     njs_slice_prop_t *slice, njs_value_t *args, njs_uint_t nargs);
 static njs_int_t njs_string_slice_args(njs_vm_t *vm, njs_slice_prop_t *slice,
     njs_value_t *args, njs_uint_t nargs);
-static njs_int_t njs_string_from_char_code(njs_vm_t *vm,
-    njs_value_t *args, njs_uint_t nargs, njs_index_t unused);
-static njs_int_t njs_string_from_code_point(njs_vm_t *vm, njs_value_t *args,
-    njs_uint_t nargs, njs_index_t unused);
+static njs_int_t njs_string_from_char_code(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t is_point);
 static njs_int_t njs_string_bytes_from(njs_vm_t *vm, njs_value_t *args,
     njs_uint_t nargs, njs_index_t unused);
 static njs_int_t njs_string_bytes_from_array_like(njs_vm_t *vm,
@@ -545,7 +543,7 @@ static const njs_object_prop_t  njs_stri
     {
         .type = NJS_PROPERTY,
         .name = njs_string("fromCharCode"),
-        .value = njs_native_function(njs_string_from_char_code, 1),
+        .value = njs_native_function2(njs_string_from_char_code, 1, 0),
         .writable = 1,
         .configurable = 1,
     },
@@ -553,7 +551,7 @@ static const njs_object_prop_t  njs_stri
     {
         .type = NJS_PROPERTY,
         .name = njs_string("fromCodePoint"),
-        .value = njs_native_function(njs_string_from_code_point, 1),
+        .value = njs_native_function2(njs_string_from_char_code, 1, 1),
         .writable = 1,
         .configurable = 1,
     },
@@ -1029,13 +1027,14 @@ static njs_int_t
 njs_string_prototype_to_bytes(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     njs_index_t unused)
 {
-    u_char             *p;
-    size_t             length;
-    uint32_t           byte;
-    njs_int_t          ret;
-    const u_char       *s, *end;
-    njs_slice_prop_t   slice;
-    njs_string_prop_t  string;
+    u_char                *p;
+    size_t                length;
+    uint32_t              byte;
+    njs_int_t             ret;
+    const u_char          *s, *end;
+    njs_slice_prop_t      slice;
+    njs_string_prop_t     string;
+    njs_unicode_decode_t  ctx;
 
     ret = njs_string_object_validate(vm, njs_arg(args, nargs, 0));
     if (njs_slow_path(ret != NJS_OK)) {
@@ -1064,8 +1063,10 @@ njs_string_prototype_to_bytes(njs_vm_t *
 
             length = slice.length;
 
+            njs_utf8_decode_init(&ctx);
+
             while (length != 0 && s < end) {
-                byte = njs_utf8_decode(&s, end);
+                byte = njs_utf8_decode(&ctx, &s, end);
 
                 if (njs_slow_path(byte > 0xFF)) {
                     njs_release(vm, &vm->retval);
@@ -1463,13 +1464,14 @@ static njs_int_t
 njs_string_prototype_char_code_at(njs_vm_t *vm, njs_value_t *args,
     njs_uint_t nargs, njs_index_t unused)
 {
-    double             num;
-    size_t             length;
-    int64_t            index;
-    uint32_t           code;
-    njs_int_t          ret;
-    const u_char       *start, *end;
-    njs_string_prop_t  string;
+    double                num;
+    size_t                length;
+    int64_t               index;
+    uint32_t              code;
+    njs_int_t             ret;
+    const u_char          *start, *end;
+    njs_string_prop_t     string;
+    njs_unicode_decode_t  ctx;
 
     ret = njs_string_object_validate(vm, njs_arg(args, nargs, 0));
     if (njs_slow_path(ret != NJS_OK)) {
@@ -1493,10 +1495,12 @@ njs_string_prototype_char_code_at(njs_vm
         code = string.start[index];
 
     } else {
+        njs_utf8_decode_init(&ctx);
+
         /* UTF-8 string. */
         end = string.start + string.size;
         start = njs_string_offset(string.start, end, index);
-        code = njs_utf8_decode(&start, end);
+        code = njs_utf8_decode(&ctx, &start, end);
     }
 
     num = code;
@@ -1829,14 +1833,27 @@ njs_decode_base64_core(njs_vm_t *vm, njs
 
 
 static njs_int_t
-njs_string_from_char_code(njs_vm_t *vm, njs_value_t *args,
-    njs_uint_t nargs, njs_index_t unused)
+njs_string_from_char_code(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t is_point)
 {
-    u_char      *p;
-    size_t      size;
-    uint16_t    code;
-    njs_int_t   ret;
-    njs_uint_t  i;
+    double                num;
+    u_char                *p, *start, *end;
+    ssize_t               len;
+    int32_t               code;
+    uint32_t              cp;
+    uint64_t              length, size;
+    njs_int_t             ret;
+    njs_uint_t            i;
+    njs_unicode_decode_t  ctx;
+    u_char                buf[4];
+
+    size = 0;
+    length = 0;
+
+    cp = 0x00;
+    end = buf + sizeof(buf);
+
+    njs_utf16_decode_init(&ctx);
 
     for (i = 1; i < nargs; i++) {
         if (!njs_is_numeric(&args[i])) {
@@ -1845,73 +1862,76 @@ njs_string_from_char_code(njs_vm_t *vm, 
                 return ret;
             }
         }
+
+        if (is_point) {
+            num = njs_number(&args[i]);
+            if (isnan(num)) {
+                goto range_error;
+            }
+
+            code = num;
+
+            if (code != num || code < 0 || code > 0x10FFFF) {
+                goto range_error;
+            }
+
+        } else {
+            code = njs_number_to_uint16(njs_number(&args[i]));
+        }
+
+        start = buf;
+        len = njs_utf16_encode(code, &start, end);
+
+        start = buf;
+        cp = njs_utf16_decode(&ctx, (const u_char **) &start, start + len);
+
+        if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+            if (cp == NJS_UNICODE_CONTINUE) {
+                continue;
+            }
+
+            cp = NJS_UNICODE_REPLACEMENT;
+        }
+
+        size += njs_utf8_size(cp);
+        length++;
     }
 
-    size = 0;
-
-    for (i = 1; i < nargs; i++) {
-        code = njs_number_to_uint16(njs_number(&args[i]));
-        size += njs_utf8_size_uint16(code);
+    if (cp == NJS_UNICODE_CONTINUE) {
+        size += njs_utf8_size(NJS_UNICODE_REPLACEMENT);
+        length++;
     }
 
-    p = njs_string_alloc(vm, &vm->retval, size, nargs - 1);
+    p = njs_string_alloc(vm, &vm->retval, size, length);
     if (njs_slow_path(p == NULL)) {
         return NJS_ERROR;
     }
 
-    for (i = 1; i < nargs; i++) {
-        code = njs_number_to_uint16(njs_number(&args[i]));
-        p = njs_utf8_encode(p, code);
-    }
-
-    return NJS_OK;
-}
-
-
-static njs_int_t
-njs_string_from_code_point(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
-    njs_index_t unused)
-{
-    u_char      *p;
-    double      num;
-    size_t      size;
-    int32_t     code;
-    njs_int_t   ret;
-    njs_uint_t  i;
+    njs_utf16_decode_init(&ctx);
 
     for (i = 1; i < nargs; i++) {
-        if (!njs_is_numeric(&args[i])) {
-            ret = njs_value_to_numeric(vm, &args[i], &args[i]);
-            if (ret != NJS_OK) {
-                return ret;
-            }
-        }
-    }
-
-    size = 0;
-
-    for (i = 1; i < nargs; i++) {
-        num = njs_number(&args[i]);
-        if (isnan(num)) {
-            goto range_error;
+        if (is_point) {
+            code = njs_number(&args[i]);
+
+        } else {
+            code = njs_number_to_uint16(njs_number(&args[i]));
         }
 
-        code = num;
-
-        if (code != num || code < 0 || code >= 0x110000) {
-            goto range_error;
+        start = buf;
+        len = njs_utf16_encode(code, &start, end);
+
+        start = buf;
+        cp = njs_utf16_decode(&ctx, (const u_char **) &start, start + len);
+
+        if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+            if (cp == NJS_UNICODE_CONTINUE && i + 1 != nargs) {
+                continue;
+            }
+
+            cp = NJS_UNICODE_REPLACEMENT;
         }
 
-        size += njs_utf8_size(code);
-    }
-
-    p = njs_string_alloc(vm, &vm->retval, size, nargs - 1);
-    if (njs_slow_path(p == NULL)) {
-        return NJS_ERROR;
-    }
-
-    for (i = 1; i < nargs; i++) {
-        p = njs_utf8_encode(p, njs_number(&args[i]));
+        p = njs_utf8_encode(p, cp);
     }
 
     return NJS_OK;
@@ -2591,11 +2611,12 @@ static njs_int_t
 njs_string_prototype_trim(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     njs_index_t mode)
 {
-    uint32_t           u, trim, length;
-    njs_int_t          ret;
-    njs_value_t        *value;
-    const u_char       *p, *prev, *start, *end;
-    njs_string_prop_t  string;
+    uint32_t              u, trim, length;
+    njs_int_t             ret;
+    njs_value_t           *value;
+    const u_char          *p, *prev, *start, *end;
+    njs_string_prop_t     string;
+    njs_unicode_decode_t  ctx;
 
     value = njs_argument(args, 0);
     ret = njs_string_object_validate(vm, value);
@@ -2651,13 +2672,15 @@ njs_string_prototype_trim(njs_vm_t *vm, 
         /* UTF-8 string. */
 
         if (mode & NJS_TRIM_START) {
+            njs_utf8_decode_init(&ctx);
+
             for ( ;; ) {
                 if (start == end) {
                     goto empty;
                 }
 
                 p = start;
-                u = njs_utf8_decode(&start, end);
+                u = njs_utf8_decode(&ctx, &start, end);
 
                 if (njs_utf8_is_whitespace(u)) {
                     trim++;
@@ -2672,6 +2695,8 @@ njs_string_prototype_trim(njs_vm_t *vm, 
         if (mode & NJS_TRIM_END) {
             prev = end;
 
+            njs_utf8_decode_init(&ctx);
+
             for ( ;; ) {
                 if (start == prev) {
                     goto empty;
@@ -2679,7 +2704,7 @@ njs_string_prototype_trim(njs_vm_t *vm, 
 
                 prev = njs_utf8_prev(prev);
                 p = prev;
-                u = njs_utf8_decode(&p, end);
+                u = njs_utf8_decode(&ctx, &p, end);
 
                 if (njs_utf8_is_whitespace(u)) {
                     trim++;
@@ -3640,11 +3665,12 @@ njs_string_prototype_replace(njs_vm_t *v
 double
 njs_string_to_number(const njs_value_t *value, njs_bool_t parse_float)
 {
-    double        num;
-    size_t        size;
-    uint32_t      u;
-    njs_bool_t    minus;
-    const u_char  *p, *start, *end;
+    double                num;
+    size_t                size;
+    uint32_t              u;
+    njs_bool_t            minus;
+    const u_char          *p, *start, *end;
+    njs_unicode_decode_t  ctx;
 
     const size_t  infinity = njs_length("Infinity");
 
@@ -3660,9 +3686,11 @@ njs_string_to_number(const njs_value_t *
 
     end = p + size;
 
+    njs_utf8_decode_init(&ctx);
+
     while (p < end) {
         start = p;
-        u = njs_utf8_decode(&p, end);
+        u = njs_utf8_decode(&ctx, &p, end);
 
         if (!njs_utf8_is_whitespace(u)) {
             p = start;
@@ -4179,15 +4207,16 @@ njs_int_t
 njs_string_encode_uri(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     njs_index_t component)
 {
-    u_char             byte, *dst;
-    uint64_t           size;
-    uint32_t           cp, cp_low;
-    njs_int_t          ret;
-    njs_value_t        *value;
-    const u_char       *src, *end;
-    const uint32_t     *escape;
-    njs_string_prop_t  string;
-    u_char             encode[4];
+    u_char                byte, *dst;
+    uint64_t              size;
+    uint32_t              cp, cp_low;
+    njs_int_t             ret;
+    njs_value_t           *value;
+    const u_char          *src, *end;
+    const uint32_t        *escape;
+    njs_string_prop_t     string;
+    njs_unicode_decode_t  ctx;
+    u_char                encode[4];
 
     static const uint32_t  escape_uri[] = {
         0xffffffff,  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
@@ -4257,8 +4286,10 @@ njs_string_encode_uri(njs_vm_t *vm, njs_
     } else {
         /* UTF-8 string. */
 
+        njs_utf8_decode_init(&ctx);
+
         while (src < end) {
-            cp = njs_utf8_decode(&src, end);
+            cp = njs_utf8_decode(&ctx, &src, end);
 
             if (cp < 0x80 && !njs_need_escape(escape, cp)) {
                 size++;
@@ -4271,7 +4302,7 @@ njs_string_encode_uri(njs_vm_t *vm, njs_
                 }
 
                 if (njs_surrogate_leading(cp)) {
-                    cp_low = njs_utf8_decode(&src, end);
+                    cp_low = njs_utf8_decode(&ctx, &src, end);
 
                     if (njs_slow_path(!njs_surrogate_trailing(cp_low))) {
                         goto uri_error;
@@ -4310,11 +4341,13 @@ njs_string_encode_uri(njs_vm_t *vm, njs_
 
     /* UTF-8 string. */
 
+    njs_utf8_decode_init(&ctx);
+
     while (src < end) {
-        cp = njs_utf8_decode(&src, end);
+        cp = njs_utf8_decode(&ctx, &src, end);
 
         if (njs_slow_path(njs_surrogate_leading(cp))) {
-            cp_low = njs_utf8_decode(&src, end);
+            cp_low = njs_utf8_decode(&ctx, &src, end);
             cp = njs_string_surrogate_pair(cp, cp_low);
         }
 
@@ -4337,11 +4370,14 @@ njs_inline uint32_t
 njs_string_decode_uri_cp(const int8_t *hex, const u_char **start,
     const u_char *end, njs_bool_t expect_percent)
 {
-    int8_t        d0, d1;
-    uint32_t      cp;
-    const u_char  *p;
-
-    cp = njs_utf8_decode(start, end);
+    int8_t                d0, d1;
+    uint32_t              cp;
+    const u_char          *p;
+    njs_unicode_decode_t  ctx;
+
+    njs_utf8_decode_init(&ctx);
+
+    cp = njs_utf8_decode(&ctx, start, end);
     if (njs_fast_path(cp != '%')) {
         return expect_percent ? 0xFFFFFFFF: cp;
     }
@@ -4378,18 +4414,19 @@ njs_int_t
 njs_string_decode_uri(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     njs_index_t component)
 {
-    u_char             *dst;
-    int64_t            size, length;
-    uint32_t           cp;
-    njs_int_t          ret;
-    njs_chb_t          chain;
-    njs_uint_t         i, n;
-    njs_bool_t         percent;
-    njs_value_t        *value;
-    const u_char       *src, *p, *end;
-    const uint32_t     *reserve;
-    njs_string_prop_t  string;
-    u_char             encode[4];
+    u_char                *dst;
+    int64_t               size, length;
+    uint32_t              cp;
+    njs_int_t             ret;
+    njs_chb_t             chain;
+    njs_uint_t            i, n;
+    njs_bool_t            percent;
+    njs_value_t           *value;
+    const u_char          *src, *p, *end;
+    const uint32_t        *reserve;
+    njs_string_prop_t     string;
+    njs_unicode_decode_t  ctx;
+    u_char                encode[4];
 
     static const uint32_t  reserve_uri[] = {
         0x00000000,  /* 0000 0000 0000 0000  0000 0000 0000 0000 */
@@ -4472,6 +4509,8 @@ njs_string_decode_uri(njs_vm_t *vm, njs_
 
     njs_chb_init(&chain, vm->mem_pool);
 
+    njs_utf8_decode_init(&ctx);
+
     while (src < end) {
         percent = (src[0] == '%');
         cp = njs_string_decode_uri_cp(hex, &src, end, 0);
@@ -4529,8 +4568,8 @@ njs_string_decode_uri(njs_vm_t *vm, njs_
         }
 
         p = encode;
-        cp = njs_utf8_decode(&p, p + n);
-        if (njs_slow_path(cp == 0xFFFFFFFF)) {
+        cp = njs_utf8_decode(&ctx, &p, p + n);
+        if (njs_slow_path(cp > NJS_UNICODE_MAX_CODEPOINT)) {
             goto uri_error;
         }
 
diff -r 63106bd2e9bf -r 855edd76bdb6 src/njs_unicode.h
--- a/src/njs_unicode.h	Wed Jul 15 19:19:18 2020 +0300
+++ b/src/njs_unicode.h	Wed Jul 15 19:19:19 2020 +0300
@@ -9,6 +9,7 @@
 
 
 enum {
+    NJS_UNICODE_REPLACEMENT   = 0xFFFD,
     NJS_UNICODE_MAX_CODEPOINT = 0x10FFFF,
     NJS_UNICODE_ERROR         = 0x1FFFFF,
     NJS_UNICODE_CONTINUE      = 0x2FFFFF
@@ -16,6 +17,9 @@ enum {
 
 typedef struct {
     uint32_t  codepoint;
+
+    unsigned  need;
+    u_char    lower;
     u_char    upper;
 } njs_unicode_decode_t;
 
diff -r 63106bd2e9bf -r 855edd76bdb6 src/njs_utf8.c
--- a/src/njs_utf8.c	Wed Jul 15 19:19:18 2020 +0300
+++ b/src/njs_utf8.c	Wed Jul 15 19:19:19 2020 +0300
@@ -56,211 +56,166 @@ njs_utf8_encode(u_char *p, uint32_t u)
 }
 
 
-/*
- * njs_utf8_decode() decodes UTF-8 sequences and returns a valid
- * character 0x00 - 0x10FFFF, or 0xFFFFFFFF for invalid or overlong
- * UTF-8 sequence.
- */
+njs_inline njs_int_t
+njs_utf8_boundary(njs_unicode_decode_t *ctx, const u_char **data,
+    unsigned *need, u_char lower, u_char upper)
+{
+    u_char  ch;
 
-uint32_t
-njs_utf8_decode(const u_char **start, const u_char *end)
-{
-    uint32_t  u;
+    ch = **data;
 
-    u = (uint32_t) **start;
-
-    if (u < 0x80) {
-        (*start)++;
-        return u;
+    if (ch < lower || ch > upper) {
+        return NJS_ERROR;
     }
 
-    return njs_utf8_decode2(start, end);
+    (*data)++;
+    (*need)--;
+    ctx->codepoint = (ctx->codepoint << 6) | (ch & 0x3F);
+
+    return NJS_OK;
 }
 
 
-/*
- * njs_utf8_decode2() decodes two and more bytes UTF-8 sequences only
- * and returns a valid character 0x80 - 0x10FFFF, OR 0xFFFFFFFF for
- * invalid or overlong UTF-8 sequence.
- */
-
-uint32_t
-njs_utf8_decode2(const u_char **start, const u_char *end)
+njs_inline void
+njs_utf8_boundary_set(njs_unicode_decode_t *ctx, const u_char ch,
+    u_char first, u_char second, u_char lower, u_char upper)
 {
-    u_char        c;
-    size_t        n;
-    uint32_t      u, overlong;
-    const u_char  *p;
-
-    p = *start;
-    u = (uint32_t) *p;
-
-    if (u >= 0xE0) {
-
-        if (u >= 0xF0) {
-
-            if (njs_slow_path(u > 0xF4)) {
-                /*
-                 * The maximum valid Unicode character is 0x10FFFF
-                 * which is encoded as 0xF4 0x8F 0xBF 0xBF.
-                 */
-                return 0xFFFFFFFF;
-            }
-
-            u &= 0x07;
-            overlong = 0x00FFFF;
-            n = 3;
-
-        } else {
-            u &= 0x0F;
-            overlong = 0x07FF;
-            n = 2;
-        }
+    if (ch == first) {
+        ctx->lower = lower;
+        ctx->upper = 0xBF;
 
-    } else if (u >= 0xC2) {
-
-        /* 0x80 is encoded as 0xC2 0x80. */
-
-        u &= 0x1F;
-        overlong = 0x007F;
-        n = 1;
-
-    } else {
-        /* u <= 0xC2 */
-        return 0xFFFFFFFF;
+    } else if (ch == second) {
+        ctx->lower = 0x80;
+        ctx->upper = upper;
     }
-
-    p++;
-
-    if (njs_fast_path(p + n <= end)) {
-
-        do {
-            c = *p++;
-            /*
-             * The byte must in the 0x80 - 0xBF range.
-             * Values below 0x80 become >= 0x80.
-             */
-            c = c - 0x80;
-
-            if (njs_slow_path(c > 0x3F)) {
-                return 0xFFFFFFFF;
-            }
-
-            u = (u << 6) | c;
-            n--;
-
-        } while (n != 0);
-
-        if (overlong < u && u < 0x110000) {
-            *start = p;
-            return u;
-        }
-    }
-
-    return 0xFFFFFFFF;
 }
 
 
 uint32_t
-njs_utf8_safe_decode(const u_char **start, const u_char *end)
-{
-    uint32_t  u;
-
-    u = (uint32_t) **start;
-
-    if (u < 0x80) {
-        (*start)++;
-        return u;
-    }
-
-    return njs_utf8_safe_decode2(start, end);
-}
-
-
-uint32_t
-njs_utf8_safe_decode2(const u_char **start, const u_char *end)
+njs_utf8_decode(njs_unicode_decode_t *ctx, const u_char **start,
+    const u_char *end)
 {
     u_char        c;
-    size_t        n;
-    uint32_t      u, overlong;
+    unsigned      need;
+    njs_int_t     ret;
     const u_char  *p;
 
-    p = *start;
-    u = (uint32_t) *p;
-
-    if (u >= 0xE0) {
-
-        if (u >= 0xF0) {
+    if (ctx->need != 0) {
+        need = ctx->need;
+        ctx->need = 0;
 
-            if (njs_slow_path(u > 0xF4)) {
-                /*
-                 * The maximum valid Unicode character is 0x10FFFF
-                 * which is encoded as 0xF4 0x8F 0xBF 0xBF.
-                 */
-                goto fail_one;
+        if (ctx->lower != 0x00) {
+            ret = njs_utf8_boundary(ctx, start, &need, ctx->lower, ctx->upper);
+            if (njs_slow_path(ret != NJS_OK)) {
+                goto failed;
             }
 
-            u &= 0x07;
-            overlong = 0x00FFFF;
-            n = 3;
+            ctx->lower = 0x00;
+        }
+
+        goto decode;
+    }
+
+    c = *(*start)++;
+
+    if (c < 0x80) {
+        return c;
 
-        } else {
-            u &= 0x0F;
-            overlong = 0x07FF;
-            n = 2;
+    } else if (c <= 0xDF) {
+        if (c < 0xC2) {
+            return NJS_UNICODE_ERROR;
+        }
+
+        need = 1;
+        ctx->codepoint = c & 0x1F;
+
+    } else if (c < 0xF0) {
+        need = 2;
+        ctx->codepoint = c & 0x0F;
+
+        if (*start == end) {
+            njs_utf8_boundary_set(ctx, c, 0xE0, 0xED, 0xA0, 0x9F);
+            goto next;
         }
 
-    } else if (u >= 0xC2) {
+        ret = NJS_OK;
+
+        if (c == 0xE0) {
+            ret = njs_utf8_boundary(ctx, start, &need, 0xA0, 0xBF);
 
-        /* 0x80 is encoded as 0xC2 0x80. */
+        } else if (c == 0xED) {
+            ret = njs_utf8_boundary(ctx, start, &need, 0x80, 0x9F);
+        }
+
+        if (njs_slow_path(ret != NJS_OK)) {
+            goto failed;
+        }
+
+    } else if (c < 0xF5) {
+        need = 3;
+        ctx->codepoint = c & 0x07;
 
-        u &= 0x1F;
-        overlong = 0x007F;
-        n = 1;
+        if (*start == end) {
+            njs_utf8_boundary_set(ctx, c, 0xF0, 0xF4, 0x90, 0x8F);
+            goto next;
+        }
+
+        ret = NJS_OK;
+
+        if (c == 0xF0) {
+            ret = njs_utf8_boundary(ctx, start, &need, 0x90, 0xBF);
+
+        } else if (c == 0xF4) {
+            ret = njs_utf8_boundary(ctx, start, &need, 0x80, 0x8F);
+        }
+
+        if (njs_slow_path(ret != NJS_OK)) {
+            goto failed;
+        }
 
     } else {
-        /* u <= 0xC2 */
-        goto fail_one;
+        return NJS_UNICODE_ERROR;
     }
 
-    p++;
+decode:
+
+    for (p = *start; p < end; p++) {
+        c = *p;
 
-    while (p < end && n != 0) {
-        c = *p++;
-        /*
-         * The byte must in the 0x80 - 0xBF range.
-         * Values below 0x80 become >= 0x80.
-         */
-        c = c - 0x80;
+        if (c < 0x80 || c > 0xBF) {
+            *start = p;
 
-        if (njs_slow_path(c > 0x3F)) {
-            *start = --p;
-            return NJS_UTF8_REPLACEMENT;
+            goto failed;
         }
 
-        u = (u << 6) | c;
-        n--;
+        ctx->codepoint = (ctx->codepoint << 6) | (c & 0x3F);
+
+        if (--need == 0) {
+            *start = p + 1;
+
+            return ctx->codepoint;
+        }
     }
 
     *start = p;
 
-    if (n == 0 && overlong < u && u < 0x110000) {
-        return u;
-    }
+next:
 
-    return NJS_UTF8_REPLACEMENT;
+    ctx->need = need;

From weijunjie at didiglobal.com  Fri Jul 17 08:50:44 2020
From: weijunjie at didiglobal.com (=?gb2312?B?zrq/ob3cIEp1bmppZSBXZWk=?=)
Date: Fri, 17 Jul 2020 08:50:44 +0000
Subject: Thrift proxy code contribution
Message-ID: <1594975801095.50356@didiglobal.com>

Hello!


Thanks for your reply!


> I don't see any problems with publishing your code as a module for
> nginx.  Changes you've mentioned in nginx itself should be
> relatively easy to avoid.

Do you mean publishing as third party modules like modules in https://www.nginx.com/resources/wiki/modules/ ?


Thanks!

Junjie Wei

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200717/4b567448/attachment.htm>

From agentzh at gmail.com  Fri Jul 17 17:41:46 2020
From: agentzh at gmail.com (Yichun Zhang (agentzh))
Date: Fri, 17 Jul 2020 10:41:46 -0700
Subject: [ANN] Test::Nginx 0.28 is released
Message-ID: <CAB4Tn6Np6fdmk-UJAN23NFYPFTWFWfY__Qnuj=FqwjXocUSBhQ@mail.gmail.com>

Hi there,

I am happy to announce the new 0.28 release of Test::Nginx:

https://openresty.org/en/ann-test-nginx-028.html

This version fixes the Test2::Util module dependency problem
introduced in the previous 0.27 release.

This Perl module provides a test scaffold for automated testing in
Nginx C module or OpenResty-based Lua library development and
regression testing.

This class inherits from Test::Base, thus bringing all its declarative
power to the Nginx C module testing practices.

All of our OpenResty projects are using this test scaffold for
automated regression testing.

Enjoy!

Best regards,
Yichun
---
Yichun Zhang is the creator of OpenResty, the founder and CEO of OpenResty Inc.

From balus at foxmail.com  Sat Jul 18 12:09:30 2020
From: balus at foxmail.com (=?ISO-8859-1?B?YmFsdXM=?=)
Date: Sat, 18 Jul 2020 20:09:30 +0800
Subject: [PATCH] Core: enclosed parameters of macros in parentheses.
Message-ID: <tencent_14CA823123E68B9E69704D9CE236EDB4A309@qq.com>

Hi,


I have been writing a nginx module recently, in which I use
ngx_buf_t to store data received from network, such as:


static u_char&nbsp; &nbsp; &nbsp;buffer[4096];
static ngx_buf_t&nbsp; b;


b.pos = b.last = b.start = buffer;
b.end = b.start + sizeof(buffer);


...


size = ngx_buf_size(&amp;b);


then I find it not work since the parameter in ngx_buf_size is not
closed by parenthese. And I think this is a common situation so I
made this patch:


# HG changeset patch
# User balus <balus at foxmail.com&gt;
# Date 1595073121 -28800
#&nbsp; &nbsp; &nbsp; Sat Jul 18 19:52:01 2020 +0800
# Node ID 92d9878c0c7549345f0a144cd81a6b6d45f21fc6
# Parent&nbsp; 32a343635b50662979975e1204417bb1fc7e1b1f
Core: enclosed parameters of macros in parentheses.


diff -r 32a343635b50 -r 92d9878c0c75 src/core/ngx_buf.h
--- a/src/core/ngx_buf.h	Thu Jul 09 16:21:37 2020 +0300
+++ b/src/core/ngx_buf.h	Sat Jul 18 19:52:01 2020 +0800
@@ -125,20 +125,22 @@
&nbsp;#define NGX_CHAIN_ERROR&nbsp; &nbsp; &nbsp;(ngx_chain_t *) NGX_ERROR
&nbsp;
&nbsp;
-#define ngx_buf_in_memory(b)&nbsp; &nbsp; &nbsp; &nbsp; (b-&gt;temporary || b-&gt;memory || b-&gt;mmap)
-#define ngx_buf_in_memory_only(b)&nbsp; &nbsp;(ngx_buf_in_memory(b) &amp;&amp; !b-&gt;in_file)
+#define ngx_buf_in_memory(b)&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;\
+&nbsp; &nbsp; ((b)-&gt;temporary || (b)-&gt;memory || (b)-&gt;mmap)
+#define ngx_buf_in_memory_only(b)&nbsp; &nbsp;(ngx_buf_in_memory(b) &amp;&amp; !(b)-&gt;in_file)
&nbsp;
&nbsp;#define ngx_buf_special(b)&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;\
-&nbsp; &nbsp; ((b-&gt;flush || b-&gt;last_buf || b-&gt;sync)&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; \
-&nbsp; &nbsp; &nbsp;&amp;&amp; !ngx_buf_in_memory(b) &amp;&amp; !b-&gt;in_file)
+&nbsp; &nbsp; (((b)-&gt;flush || (b)-&gt;last_buf || (b)-&gt;sync)&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; \
+&nbsp; &nbsp; &nbsp;&amp;&amp; !ngx_buf_in_memory(b) &amp;&amp; !(b)-&gt;in_file)
&nbsp;
&nbsp;#define ngx_buf_sync_only(b)&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;\
-&nbsp; &nbsp; (b-&gt;sync&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;\
-&nbsp; &nbsp; &nbsp;&amp;&amp; !ngx_buf_in_memory(b) &amp;&amp; !b-&gt;in_file &amp;&amp; !b-&gt;flush &amp;&amp; !b-&gt;last_buf)
+&nbsp; &nbsp; ((b)-&gt;sync&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;\
+&nbsp; &nbsp; &nbsp;&amp;&amp; !ngx_buf_in_memory(b)&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; \
+&nbsp; &nbsp; &nbsp;&amp;&amp; !(b)-&gt;in_file &amp;&amp; !(b)-&gt;flush &amp;&amp; !(b)-&gt;last_buf)
&nbsp;
&nbsp;#define ngx_buf_size(b)&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; \
-&nbsp; &nbsp; (ngx_buf_in_memory(b) ? (off_t) (b-&gt;last - b-&gt;pos):&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; \
-&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; (b-&gt;file_last - b-&gt;file_pos))
+&nbsp; &nbsp; (ngx_buf_in_memory(b) ? (off_t) ((b)-&gt;last - (b)-&gt;pos)&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;\
+&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; : ((b)-&gt;file_last - (b)-&gt;file_pos))
&nbsp;
&nbsp;ngx_buf_t *ngx_create_temp_buf(ngx_pool_t *pool, size_t size);
&nbsp;ngx_chain_t *ngx_create_chain_of_bufs(ngx_pool_t *pool, ngx_bufs_t *bufs);
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200718/cb2d6df5/attachment.htm>

From brytemorio at qngnode.cc  Mon Jul 20 16:43:15 2020
From: brytemorio at qngnode.cc (Bryte Morio)
Date: Mon, 20 Jul 2020 17:43:15 +0100
Subject: ASGI Protocol
Message-ID: <d36fec19-7562-c567-cf3f-c39eee92a95e@qngnode.cc>

Will there be future native support for the asynchronous server gateway 
interface? protocol for reverse-proxy, because it is not covered 
anywhere in the? current documentation.


From mdounin at mdounin.ru  Tue Jul 21 13:41:29 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 21 Jul 2020 16:41:29 +0300
Subject: ASGI Protocol
In-Reply-To: <d36fec19-7562-c567-cf3f-c39eee92a95e@qngnode.cc>
References: <d36fec19-7562-c567-cf3f-c39eee92a95e@qngnode.cc>
Message-ID: <20200721134129.GJ12747@mdounin.ru>

Hello!

On Mon, Jul 20, 2020 at 05:43:15PM +0100, Bryte Morio wrote:

> Will there be future native support for the asynchronous server 
> gateway interface? protocol for reverse-proxy, because it is not 
> covered anywhere in the?
> current documentation.

For now there are no plans to support ASGI protocol.  Given that the 
question is on the nginx-devel@ mailing list: don't hesitate to 
ask questions if you need help with development of your module to 
support ASGI.

-- 
Maxim Dounin
http://mdounin.ru/

From harishkumarivaturi at gmail.com  Tue Jul 21 14:03:02 2020
From: harishkumarivaturi at gmail.com (HARISH KUMAR Ivaturi)
Date: Tue, 21 Jul 2020 16:03:02 +0200
Subject: Nginx support for HTTP/3
Message-ID: <CAGmfrBzhN-zJidBJTYySnMO=Uz2uhP_7U84EbUuWrZ8o6i9tjw@mail.gmail.com>

Hi

I am Harish Kumar, Master Student at BTH Sweden. I am doing my thesis on
OpenStack with HTTP/3 and dependencies to complete my thesis are Nginx Web
Server with HTTP/3 feature and curl with HTTP/3. I have successfully built
curl but w.r.t Nginx with HTTP/3 i could not.

There are two sources which i found to build Nginx with HTTP/3 and these
sources are mentioned below:
1)
https://blog.cloudflare.com/experiment-with-http-3-using-nginx-and-quiche/
2) https://quic.nginx.org/readme.html

I tried both and could not build nginx with HTTP/3. I get a lot of errors
like openssl with QUIC support is not there though HTTP/3 is built using
BoringSSL and I am not able to get a clarity. I request you to please give
me some guidance and suggestions on  how to build Nginx with HTTP/3 , i
followed both official sources but i failed. Unless i have a Nginx web
server with HTTP/3 enabled I cannot move further with my Thesis.

Please let me know.

BR
Harish Kumar
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200721/3b4a698e/attachment.htm>

From manuel.baesler at gmail.com  Tue Jul 21 15:39:53 2020
From: manuel.baesler at gmail.com (Manuel)
Date: Tue, 21 Jul 2020 17:39:53 +0200
Subject: Nginx support for HTTP/3
In-Reply-To: <CAGmfrBzhN-zJidBJTYySnMO=Uz2uhP_7U84EbUuWrZ8o6i9tjw@mail.gmail.com>
References: <CAGmfrBzhN-zJidBJTYySnMO=Uz2uhP_7U84EbUuWrZ8o6i9tjw@mail.gmail.com>
Message-ID: <CAFe2+oe0PsRnxKiys0KzAg_1DKYn_K=XDk0=Bp9mnNj0k5=xxQ@mail.gmail.com>

Hi,

as the readme from the hg repo indicates, you first have to build the
boring lib. (build/ssl build/crypto)

  $ ./auto/configure [..]
--with-ld-opt="-L../boringssl/build/ssl -L../boringssl/build/crypto"
[..]
checking for --with-ld-opt="-L../boringssl/build/ssl
-L../boringssl/build/crypto" ... found
[..]
nginx-quic/objs$ ./nginx -V
nginx version: nginx/1.19.1
built with OpenSSL 1.1.0 (compatible; BoringSSL) (running with BoringSSL)
TLS SNI support enabled
configure arguments: --with-debug --with-http_v3_module
--with-cc-opt=-I../boringssl/include
--with-ld-opt='-L../boringssl/build/ssl -L../boringssl/build/crypto'

Am Di., 21. Juli 2020 um 16:03 Uhr schrieb HARISH KUMAR Ivaturi
<harishkumarivaturi at gmail.com>:
>
> Hi
>
> I am Harish Kumar, Master Student at BTH Sweden. I am doing my thesis on OpenStack with HTTP/3 and dependencies to complete my thesis are Nginx Web Server with HTTP/3 feature and curl with HTTP/3. I have successfully built curl but w.r.t Nginx with HTTP/3 i could not.
>
> There are two sources which i found to build Nginx with HTTP/3 and these sources are mentioned below:
> 1) https://blog.cloudflare.com/experiment-with-http-3-using-nginx-and-quiche/
> 2) https://quic.nginx.org/readme.html
>
> I tried both and could not build nginx with HTTP/3. I get a lot of errors like openssl with QUIC support is not there though HTTP/3 is built using BoringSSL and I am not able to get a clarity. I request you to please give me some guidance and suggestions on  how to build Nginx with HTTP/3 , i followed both official sources but i failed. Unless i have a Nginx web server with HTTP/3 enabled I cannot move further with my Thesis.
>
> Please let me know.
>
> BR
> Harish Kumar
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

From harishkumarivaturi at gmail.com  Tue Jul 21 16:38:48 2020
From: harishkumarivaturi at gmail.com (HARISH KUMAR Ivaturi)
Date: Tue, 21 Jul 2020 18:38:48 +0200
Subject: Nginx support for HTTP/3
In-Reply-To: <CAFe2+oe0PsRnxKiys0KzAg_1DKYn_K=XDk0=Bp9mnNj0k5=xxQ@mail.gmail.com>
References: <CAGmfrBzhN-zJidBJTYySnMO=Uz2uhP_7U84EbUuWrZ8o6i9tjw@mail.gmail.com>
 <CAFe2+oe0PsRnxKiys0KzAg_1DKYn_K=XDk0=Bp9mnNj0k5=xxQ@mail.gmail.com>
Message-ID: <CAGmfrBxRCTk_850V58XGc3MEH7hYaKY3c=A5CjO=CRF=xJBWow@mail.gmail.com>

Hi Manuel
Thanks for the reply.

i did that, but i executed the command at a time , didn't do separately.
The command executed was

$ ./auto/configure --with-debug --with-http_v3_module       \
                       --with-cc-opt="-I../boringssl/include"   \
                       --with-ld-opt="-L../boringssl/build/ssl  \
                                      -L../boringssl/build/crypto"


And later says Error: OpenSSL with QUIC support is not there.


Could you send me a video of your explanation from scratch? Building
this in your linux ubuntu.


BR

Harish Kumar


On Tue, Jul 21, 2020 at 5:40 PM Manuel <manuel.baesler at gmail.com> wrote:

> Hi,
>
> as the readme from the hg repo indicates, you first have to build the
> boring lib. (build/ssl build/crypto)
>
>   $ ./auto/configure [..]
> --with-ld-opt="-L../boringssl/build/ssl -L../boringssl/build/crypto"
> [..]
> checking for --with-ld-opt="-L../boringssl/build/ssl
> -L../boringssl/build/crypto" ... found
> [..]
> nginx-quic/objs$ ./nginx -V
> nginx version: nginx/1.19.1
> built with OpenSSL 1.1.0 (compatible; BoringSSL) (running with BoringSSL)
> TLS SNI support enabled
> configure arguments: --with-debug --with-http_v3_module
> --with-cc-opt=-I../boringssl/include
> --with-ld-opt='-L../boringssl/build/ssl -L../boringssl/build/crypto'
>
> Am Di., 21. Juli 2020 um 16:03 Uhr schrieb HARISH KUMAR Ivaturi
> <harishkumarivaturi at gmail.com>:
> >
> > Hi
> >
> > I am Harish Kumar, Master Student at BTH Sweden. I am doing my thesis on
> OpenStack with HTTP/3 and dependencies to complete my thesis are Nginx Web
> Server with HTTP/3 feature and curl with HTTP/3. I have successfully built
> curl but w.r.t Nginx with HTTP/3 i could not.
> >
> > There are two sources which i found to build Nginx with HTTP/3 and these
> sources are mentioned below:
> > 1)
> https://blog.cloudflare.com/experiment-with-http-3-using-nginx-and-quiche/
> > 2) https://quic.nginx.org/readme.html
> >
> > I tried both and could not build nginx with HTTP/3. I get a lot of
> errors like openssl with QUIC support is not there though HTTP/3 is built
> using BoringSSL and I am not able to get a clarity. I request you to please
> give me some guidance and suggestions on  how to build Nginx with HTTP/3 ,
> i followed both official sources but i failed. Unless i have a Nginx web
> server with HTTP/3 enabled I cannot move further with my Thesis.
> >
> > Please let me know.
> >
> > BR
> > Harish Kumar
> > _______________________________________________
> > nginx-devel mailing list
> > nginx-devel at nginx.org
> > http://mailman.nginx.org/mailman/listinfo/nginx-devel
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200721/e066f533/attachment.htm>

From ru at nginx.com  Tue Jul 21 17:36:47 2020
From: ru at nginx.com (Ruslan Ermilov)
Date: Tue, 21 Jul 2020 17:36:47 +0000
Subject: [nginx] Core: close PID file when writing fails.
Message-ID: <hg.4f30f75dbdf3.1595353007.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/4f30f75dbdf3
branches:  
changeset: 7685:4f30f75dbdf3
user:      Ruslan Ermilov <ru at nginx.com>
date:      Tue Jul 21 20:34:29 2020 +0300
description:
Core: close PID file when writing fails.

Reported by Jinhua Tan.

diffstat:

 src/core/ngx_cycle.c |  7 +++++--
 1 files changed, 5 insertions(+), 2 deletions(-)

diffs (35 lines):

diff -r 32a343635b50 -r 4f30f75dbdf3 src/core/ngx_cycle.c
--- a/src/core/ngx_cycle.c	Thu Jul 09 16:21:37 2020 +0300
+++ b/src/core/ngx_cycle.c	Tue Jul 21 20:34:29 2020 +0300
@@ -1009,6 +1009,7 @@ ngx_int_t
 ngx_create_pidfile(ngx_str_t *name, ngx_log_t *log)
 {
     size_t      len;
+    ngx_int_t   rc;
     ngx_uint_t  create;
     ngx_file_t  file;
     u_char      pid[NGX_INT64_LEN + 2];
@@ -1033,11 +1034,13 @@ ngx_create_pidfile(ngx_str_t *name, ngx_
         return NGX_ERROR;
     }
 
+    rc = NGX_OK;
+
     if (!ngx_test_config) {
         len = ngx_snprintf(pid, NGX_INT64_LEN + 2, "%P%N", ngx_pid) - pid;
 
         if (ngx_write_file(&file, pid, len, 0) == NGX_ERROR) {
-            return NGX_ERROR;
+            rc = NGX_ERROR;
         }
     }
 
@@ -1046,7 +1049,7 @@ ngx_create_pidfile(ngx_str_t *name, ngx_
                       ngx_close_file_n " \"%s\" failed", file.name.data);
     }
 
-    return NGX_OK;
+    return rc;
 }
 
 

From ru at nginx.com  Tue Jul 21 17:38:43 2020
From: ru at nginx.com (Ruslan Ermilov)
Date: Tue, 21 Jul 2020 20:38:43 +0300
Subject: Core: close pid file while writing it failed.
In-Reply-To: <20200622150914.GC88174@lo0.su>
References: <CAJGB9Baq0wiWjXnUTnt+-JEuqg4yRfbBc8GGWmJPS8R7+i=7WA@mail.gmail.com>
 <20200622150914.GC88174@lo0.su>
Message-ID: <20200721173843.GI49526@lo0.su>

On Mon, Jun 22, 2020 at 06:09:14PM +0300, Ruslan Ermilov wrote:
> On Thu, May 21, 2020 at 09:45:24PM +0800, Jim T wrote:
> > Hello!
> > 
> > As far as I understand it, `ngx_create_pidfile` is a function that works
> > independently. There is no action to close the pid file externally, so we
> > need to close the pid file when the writing it failed. There are also
> > reports here https://github.com/nginx/nginx/pull/52.
> > 
> > # HG changeset patch
> > # User Jinhua Tan <312841925 at qq.com>
> > # Date 1590068494 -28800
> > #      Thu May 21 21:41:34 2020 +0800
> > # Node ID 6084ea4d9a4d2ae32f3fc4e2e3b9032ab0b71e30
> > # Parent  3242f98298975e556a7e87130611ce84799fe935
> > Core: close pid file while writing it failed.
> > 
> > diff -r 3242f9829897 -r 6084ea4d9a4d src/core/ngx_cycle.c
> > --- a/src/core/ngx_cycle.c      Wed May 20 12:24:05 2020 +0800
> > +++ b/src/core/ngx_cycle.c      Thu May 21 21:41:34 2020 +0800
> > @@ -1036,6 +1036,12 @@
> >          len = ngx_snprintf(pid, NGX_INT64_LEN + 2, "%P%N", ngx_pid) - pid;
> > 
> >          if (ngx_write_file(&file, pid, len, 0) == NGX_ERROR) {
> > +
> > +            if (ngx_close_file(file.fd) == NGX_FILE_ERROR) {
> > +                ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
> > +                              ngx_close_file_n " \"%s\" failed",
> > file.name.data);
> > +            }
> > +
> >              return NGX_ERROR;
> >          }
> >      }
> > 
> > Thank you!
> 
> How's this instead?
> 
> diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c
> --- a/src/core/ngx_cycle.c
> +++ b/src/core/ngx_cycle.c
> @@ -1009,6 +1009,7 @@ ngx_int_t
>  ngx_create_pidfile(ngx_str_t *name, ngx_log_t *log)
>  {
>      size_t      len;
> +    ngx_int_t   rc;
>      ngx_uint_t  create;
>      ngx_file_t  file;
>      u_char      pid[NGX_INT64_LEN + 2];
> @@ -1033,11 +1034,13 @@ ngx_create_pidfile(ngx_str_t *name, ngx_
>          return NGX_ERROR;
>      }
>  
> +    rc = NGX_OK;
> +
>      if (!ngx_test_config) {
>          len = ngx_snprintf(pid, NGX_INT64_LEN + 2, "%P%N", ngx_pid) - pid;
>  
>          if (ngx_write_file(&file, pid, len, 0) == NGX_ERROR) {
> -            return NGX_ERROR;
> +            rc = NGX_ERROR;
>          }
>      }
>  
> @@ -1046,7 +1049,7 @@ ngx_create_pidfile(ngx_str_t *name, ngx_
>                        ngx_close_file_n " \"%s\" failed", file.name.data);
>      }
>  
> -    return NGX_OK;
> +    return rc;
>  }
>  
>  

http://hg.nginx.org/nginx/rev/4f30f75dbdf3

From manuel.baesler at gmail.com  Tue Jul 21 20:31:32 2020
From: manuel.baesler at gmail.com (Manuel)
Date: Tue, 21 Jul 2020 20:31:32 +0000
Subject: Nginx support for HTTP/3
In-Reply-To: <CAGmfrBxRCTk_850V58XGc3MEH7hYaKY3c=A5CjO=CRF=xJBWow@mail.gmail.com>
References: <CAGmfrBzhN-zJidBJTYySnMO=Uz2uhP_7U84EbUuWrZ8o6i9tjw@mail.gmail.com>
 <CAFe2+oe0PsRnxKiys0KzAg_1DKYn_K=XDk0=Bp9mnNj0k5=xxQ@mail.gmail.com>
 <CAGmfrBxRCTk_850V58XGc3MEH7hYaKY3c=A5CjO=CRF=xJBWow@mail.gmail.com>
Message-ID: <DBBC16B6-87D6-4FC9-B810-254337ED2FD3@gmail.com>

 Before you can compile nginx you have to compile the boringssl libs.

Compiling for boringssl is documented at the boringssl webpage.

> Am 21.07.2020 um 16:38 schrieb HARISH KUMAR Ivaturi <harishkumarivaturi at gmail.com>:
> 
> Hi Manuel
> Thanks for the reply.
> 
> i did that, but i executed the command at a time , didn't do separately. The command executed was 
> $ ./auto/configure --with-debug --with-http_v3_module       \
>                        --with-cc-opt="-I../boringssl/include"   \
>                        --with-ld-opt="-L../boringssl/build/ssl  \
>                                       -L../boringssl/build/crypto"
> 
> And later says Error: OpenSSL with QUIC support is not there. 
> 
> Could you send me a video of your explanation from scratch? Building this in your linux ubuntu.
> 
> BR
> Harish Kumar
> 
>> On Tue, Jul 21, 2020 at 5:40 PM Manuel <manuel.baesler at gmail.com> wrote:
>> Hi,
>> 
>> as the readme from the hg repo indicates, you first have to build the
>> boring lib. (build/ssl build/crypto)
>> 
>>   $ ./auto/configure [..]
>> --with-ld-opt="-L../boringssl/build/ssl -L../boringssl/build/crypto"
>> [..]
>> checking for --with-ld-opt="-L../boringssl/build/ssl
>> -L../boringssl/build/crypto" ... found
>> [..]
>> nginx-quic/objs$ ./nginx -V
>> nginx version: nginx/1.19.1
>> built with OpenSSL 1.1.0 (compatible; BoringSSL) (running with BoringSSL)
>> TLS SNI support enabled
>> configure arguments: --with-debug --with-http_v3_module
>> --with-cc-opt=-I../boringssl/include
>> --with-ld-opt='-L../boringssl/build/ssl -L../boringssl/build/crypto'
>> 
>> Am Di., 21. Juli 2020 um 16:03 Uhr schrieb HARISH KUMAR Ivaturi
>> <harishkumarivaturi at gmail.com>:
>> >
>> > Hi
>> >
>> > I am Harish Kumar, Master Student at BTH Sweden. I am doing my thesis on OpenStack with HTTP/3 and dependencies to complete my thesis are Nginx Web Server with HTTP/3 feature and curl with HTTP/3. I have successfully built curl but w.r.t Nginx with HTTP/3 i could not.
>> >
>> > There are two sources which i found to build Nginx with HTTP/3 and these sources are mentioned below:
>> > 1) https://blog.cloudflare.com/experiment-with-http-3-using-nginx-and-quiche/
>> > 2) https://quic.nginx.org/readme.html
>> >
>> > I tried both and could not build nginx with HTTP/3. I get a lot of errors like openssl with QUIC support is not there though HTTP/3 is built using BoringSSL and I am not able to get a clarity. I request you to please give me some guidance and suggestions on  how to build Nginx with HTTP/3 , i followed both official sources but i failed. Unless i have a Nginx web server with HTTP/3 enabled I cannot move further with my Thesis.
>> >
>> > Please let me know.
>> >
>> > BR
>> > Harish Kumar
>> > _______________________________________________
>> > nginx-devel mailing list
>> > nginx-devel at nginx.org
>> > http://mailman.nginx.org/mailman/listinfo/nginx-devel
>> _______________________________________________
>> nginx-devel mailing list
>> nginx-devel at nginx.org
>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200721/8497e619/attachment.htm>

From balan.pothula at gmail.com  Wed Jul 22 05:52:25 2020
From: balan.pothula at gmail.com (BALAJI POTHULA)
Date: Wed, 22 Jul 2020 11:22:25 +0530
Subject: Nginx support for HTTP/3
In-Reply-To: <DBBC16B6-87D6-4FC9-B810-254337ED2FD3@gmail.com>
References: <CAGmfrBzhN-zJidBJTYySnMO=Uz2uhP_7U84EbUuWrZ8o6i9tjw@mail.gmail.com>
 <CAFe2+oe0PsRnxKiys0KzAg_1DKYn_K=XDk0=Bp9mnNj0k5=xxQ@mail.gmail.com>
 <CAGmfrBxRCTk_850V58XGc3MEH7hYaKY3c=A5CjO=CRF=xJBWow@mail.gmail.com>
 <DBBC16B6-87D6-4FC9-B810-254337ED2FD3@gmail.com>
Message-ID: <CADJuz9QOMJx=C1JcM_t=aWswvXf+6aDpxaiaxL6NbbnNVstFOA@mail.gmail.com>

Dear Harish,

Please check on the below links

https://github.com/balajipothula/nginx/blob/master/compile-nginx-on-ubuntu18.sh


https://github.com/balajipothula/nginx/blob/master/compile-nginx-lua-on-ubuntu18.sh

If the above links are helpful, I am really happy.

Thanks and regards,
BALAJI.



On Wed, Jul 22, 2020 at 2:01 AM Manuel <manuel.baesler at gmail.com> wrote:

>  Before you can compile nginx you have to compile the boringssl libs.
>
> Compiling for boringssl is documented at the boringssl webpage.
>
> Am 21.07.2020 um 16:38 schrieb HARISH KUMAR Ivaturi <
> harishkumarivaturi at gmail.com>:
>
> Hi Manuel
> Thanks for the reply.
>
> i did that, but i executed the command at a time , didn't do separately.
> The command executed was
>
> $ ./auto/configure --with-debug --with-http_v3_module       \
>                        --with-cc-opt="-I../boringssl/include"   \
>                        --with-ld-opt="-L../boringssl/build/ssl  \
>                                       -L../boringssl/build/crypto"
>
>
> And later says Error: OpenSSL with QUIC support is not there.
>
>
> Could you send me a video of your explanation from scratch? Building this in your linux ubuntu.
>
>
> BR
>
> Harish Kumar
>
>
> On Tue, Jul 21, 2020 at 5:40 PM Manuel <manuel.baesler at gmail.com> wrote:
>
>> Hi,
>>
>> as the readme from the hg repo indicates, you first have to build the
>> boring lib. (build/ssl build/crypto)
>>
>>   $ ./auto/configure [..]
>> --with-ld-opt="-L../boringssl/build/ssl -L../boringssl/build/crypto"
>> [..]
>> checking for --with-ld-opt="-L../boringssl/build/ssl
>> -L../boringssl/build/crypto" ... found
>> [..]
>> nginx-quic/objs$ ./nginx -V
>> nginx version: nginx/1.19.1
>> built with OpenSSL 1.1.0 (compatible; BoringSSL) (running with BoringSSL)
>> TLS SNI support enabled
>> configure arguments: --with-debug --with-http_v3_module
>> --with-cc-opt=-I../boringssl/include
>> --with-ld-opt='-L../boringssl/build/ssl -L../boringssl/build/crypto'
>>
>> Am Di., 21. Juli 2020 um 16:03 Uhr schrieb HARISH KUMAR Ivaturi
>> <harishkumarivaturi at gmail.com>:
>> >
>> > Hi
>> >
>> > I am Harish Kumar, Master Student at BTH Sweden. I am doing my thesis
>> on OpenStack with HTTP/3 and dependencies to complete my thesis are Nginx
>> Web Server with HTTP/3 feature and curl with HTTP/3. I have successfully
>> built curl but w.r.t Nginx with HTTP/3 i could not.
>> >
>> > There are two sources which i found to build Nginx with HTTP/3 and
>> these sources are mentioned below:
>> > 1)
>> https://blog.cloudflare.com/experiment-with-http-3-using-nginx-and-quiche/
>> > 2) https://quic.nginx.org/readme.html
>> >
>> > I tried both and could not build nginx with HTTP/3. I get a lot of
>> errors like openssl with QUIC support is not there though HTTP/3 is built
>> using BoringSSL and I am not able to get a clarity. I request you to please
>> give me some guidance and suggestions on  how to build Nginx with HTTP/3 ,
>> i followed both official sources but i failed. Unless i have a Nginx web
>> server with HTTP/3 enabled I cannot move further with my Thesis.
>> >
>> > Please let me know.
>> >
>> > BR
>> > Harish Kumar
>> > _______________________________________________
>> > nginx-devel mailing list
>> > nginx-devel at nginx.org
>> > http://mailman.nginx.org/mailman/listinfo/nginx-devel
>> _______________________________________________
>> nginx-devel mailing list
>> nginx-devel at nginx.org
>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>>
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200722/cc9966b0/attachment.htm>

From xeioex at nginx.com  Wed Jul 22 11:27:41 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Wed, 22 Jul 2020 11:27:41 +0000
Subject: [njs] Introduced njs_chb_append_str() as inline function.
Message-ID: <hg.a9e536ca9b69.1595417261.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/a9e536ca9b69
branches:  
changeset: 1473:a9e536ca9b69
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Wed Jul 22 11:27:18 2020 +0000
description:
Introduced njs_chb_append_str() as inline function.

To catch erroneous passing of njs_string_prop_t (instead of njs_str_t)
in compile time.

diffstat:

 src/njs_chb.h |  10 +++++++---
 1 files changed, 7 insertions(+), 3 deletions(-)

diffs (27 lines):

diff -r 855edd76bdb6 -r a9e536ca9b69 src/njs_chb.h
--- a/src/njs_chb.h	Wed Jul 15 19:19:19 2020 +0300
+++ b/src/njs_chb.h	Wed Jul 22 11:27:18 2020 +0000
@@ -43,9 +43,6 @@ void njs_chb_destroy(njs_chb_t *chain);
 #define njs_chb_append_literal(chain, literal)                               \
     njs_chb_append0(chain, literal, njs_length(literal))
 
-#define njs_chb_append_str(chain, str)                                       \
-    njs_chb_append0(chain, (const char *) (str)->start, (str)->length)
-
 
 #define njs_chb_node_size(n) (size_t) ((n)->pos - (n)->start)
 #define njs_chb_node_room(n) (size_t) ((n)->end - (n)->pos)
@@ -61,6 +58,13 @@ njs_chb_init(njs_chb_t *chain, njs_mp_t 
 }
 
 
+njs_inline void
+njs_chb_append_str(njs_chb_t *chain, njs_str_t *str)
+{
+    njs_chb_append0(chain, (const char *) str->start, str->length);
+}
+
+
 njs_inline int64_t
 njs_chb_size(njs_chb_t *chain)
 {

From alexander.borisov at nginx.com  Wed Jul 22 12:21:57 2020
From: alexander.borisov at nginx.com (Alexander Borisov)
Date: Wed, 22 Jul 2020 12:21:57 +0000
Subject: [njs] Fixed njs_string_truncate() function for non-byte strings.
Message-ID: <hg.b96ff1d52647.1595420517.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/b96ff1d52647
branches:  
changeset: 1474:b96ff1d52647
user:      Alexander Borisov <alexander.borisov at nginx.com>
date:      Wed Jul 22 15:21:13 2020 +0300
description:
Fixed njs_string_truncate() function for non-byte strings.

diffstat:

 src/njs_fs.c     |   2 +-
 src/njs_string.c |  25 ++++++++++++++-----------
 src/njs_string.h |   2 +-
 3 files changed, 16 insertions(+), 13 deletions(-)

diffs (88 lines):

diff -r a9e536ca9b69 -r b96ff1d52647 src/njs_fs.c
--- a/src/njs_fs.c	Wed Jul 22 11:27:18 2020 +0000
+++ b/src/njs_fs.c	Wed Jul 22 15:21:13 2020 +0300
@@ -262,7 +262,7 @@ njs_fs_read_file(njs_vm_t *vm, njs_value
 
         if (njs_slow_path(data.length < size)) {
             /* Pseudo-files may return less data than declared by st_size. */
-            njs_string_truncate(&retval, data.length);
+            njs_string_truncate(&retval, data.length, length);
         }
 
         size = data.length;
diff -r a9e536ca9b69 -r b96ff1d52647 src/njs_string.c
--- a/src/njs_string.c	Wed Jul 22 11:27:18 2020 +0000
+++ b/src/njs_string.c	Wed Jul 22 15:21:13 2020 +0300
@@ -172,30 +172,33 @@ njs_string_alloc(njs_vm_t *vm, njs_value
 
 
 void
-njs_string_truncate(njs_value_t *value, uint32_t size)
+njs_string_truncate(njs_value_t *value, uint32_t size, uint32_t length)
 {
-    u_char  *dst, *src;
+    u_char    *dst, *src;
+    uint32_t  n;
 
     if (size <= NJS_STRING_SHORT) {
-        if (value->short_string.size != NJS_STRING_LONG) {
-            value->short_string.size = size;
-
-        } else {
-            value->short_string.size = size;
+        if (value->short_string.size == NJS_STRING_LONG) {
             dst = value->short_string.start;
             src = value->long_string.data->start;
 
-            while (size != 0) {
+            n = size;
+
+            while (n != 0) {
                 /* The maximum size is just 14 bytes. */
                 njs_pragma_loop_disable_vectorization;
 
                 *dst++ = *src++;
-                size--;
+                n--;
             }
         }
 
+        value->short_string.size = size;
+        value->short_string.length = length;
+
     } else {
         value->long_string.size = size;
+        value->long_string.data->length = length;
     }
 }
 
@@ -1708,7 +1711,7 @@ njs_string_decode_hex(njs_vm_t *vm, njs_
     }
 
     if (njs_slow_path((size_t) (p - dst) != (len / 2))) {
-        njs_string_truncate(value, p - dst);
+        njs_string_truncate(value, p - dst, 0);
     }
 
     return NJS_OK;
@@ -1825,7 +1828,7 @@ njs_decode_base64_core(njs_vm_t *vm, njs
     }
 
     if (njs_slow_path((size_t) (d - dst) != dst_len)) {
-        njs_string_truncate(value, d - dst);
+        njs_string_truncate(value, d - dst, 0);
     }
 
     return NJS_OK;
diff -r a9e536ca9b69 -r b96ff1d52647 src/njs_string.h
--- a/src/njs_string.h	Wed Jul 22 11:27:18 2020 +0000
+++ b/src/njs_string.h	Wed Jul 22 15:21:13 2020 +0300
@@ -173,7 +173,7 @@ njs_int_t njs_string_decode_base64(njs_v
     const njs_str_t *src);
 njs_int_t njs_string_decode_base64url(njs_vm_t *vm, njs_value_t *value,
     const njs_str_t *src);
-void njs_string_truncate(njs_value_t *value, uint32_t size);
+void njs_string_truncate(njs_value_t *value, uint32_t size, uint32_t length);
 void njs_string_copy(njs_value_t *dst, njs_value_t *src);
 njs_int_t njs_string_validate(njs_vm_t *vm, njs_string_prop_t *string,
     njs_value_t *value);

From alexander.borisov at nginx.com  Wed Jul 22 12:21:59 2020
From: alexander.borisov at nginx.com (Alexander Borisov)
Date: Wed, 22 Jul 2020 12:21:59 +0000
Subject: [njs] Fixed clearing UTF-8 context for two-byte codepoints.
Message-ID: <hg.d88b65c37ecc.1595420519.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/d88b65c37ecc
branches:  
changeset: 1475:d88b65c37ecc
user:      Alexander Borisov <alexander.borisov at nginx.com>
date:      Wed Jul 22 15:21:13 2020 +0300
description:
Fixed clearing UTF-8 context for two-byte codepoints.

diffstat:

 src/njs_utf8.h |  1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diffs (11 lines):

diff -r b96ff1d52647 -r d88b65c37ecc src/njs_utf8.h
--- a/src/njs_utf8.h	Wed Jul 22 15:21:13 2020 +0300
+++ b/src/njs_utf8.h	Wed Jul 22 15:21:13 2020 +0300
@@ -103,6 +103,7 @@ njs_inline void
 njs_utf8_decode_init(njs_unicode_decode_t *ctx)
 {
     ctx->need = 0x00;
+    ctx->lower = 0x00;
 }
 
 

From alexander.borisov at nginx.com  Wed Jul 22 12:22:01 2020
From: alexander.borisov at nginx.com (Alexander Borisov)
Date: Wed, 22 Jul 2020 12:22:01 +0000
Subject: [njs] Fixed njs_value_property() for NJS_DECLINED from prop handler.
Message-ID: <hg.436d4c253d10.1595420521.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/436d4c253d10
branches:  
changeset: 1476:436d4c253d10
user:      Alexander Borisov <alexander.borisov at nginx.com>
date:      Wed Jul 22 15:21:14 2020 +0300
description:
Fixed njs_value_property() for NJS_DECLINED from prop handler.

This ensures retval has valid value for any case except NJS_ERROR.

diffstat:

 src/njs_value.c |  8 ++++++--
 1 files changed, 6 insertions(+), 2 deletions(-)

diffs (18 lines):

diff -r d88b65c37ecc -r 436d4c253d10 src/njs_value.c
--- a/src/njs_value.c	Wed Jul 22 15:21:13 2020 +0300
+++ b/src/njs_value.c	Wed Jul 22 15:21:14 2020 +0300
@@ -1033,8 +1033,12 @@ slow_path:
             ret = prop->value.data.u.prop_handler(vm, prop, value, NULL,
                                                   &prop->value);
 
-            if (njs_slow_path(ret == NJS_ERROR)) {
-                return ret;
+            if (njs_slow_path(ret != NJS_OK)) {
+                if (ret == NJS_ERROR) {
+                    return ret;
+                }
+
+                njs_set_undefined(&prop->value);
             }
 
             *retval = prop->value;

From alexander.borisov at nginx.com  Wed Jul 22 12:22:03 2020
From: alexander.borisov at nginx.com (Alexander Borisov)
Date: Wed, 22 Jul 2020 12:22:03 +0000
Subject: [njs] Introduced Query String module implementation.
Message-ID: <hg.7f5c5a425d03.1595420523.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/7f5c5a425d03
branches:  
changeset: 1477:7f5c5a425d03
user:      Alexander Borisov <alexander.borisov at nginx.com>
date:      Wed Jul 22 15:21:15 2020 +0300
description:
Introduced Query String module implementation.

In collaboration with Dmitry Volyntsev.

This closes #288 issue on GitHub.

diffstat:

 auto/sources             |    1 +
 src/njs_builtin.c        |    1 +
 src/njs_main.h           |    1 +
 src/njs_query_string.c   |  912 +++++++++++++++++++++++++++++++++++++++++++++++
 src/njs_query_string.h   |   12 +
 src/njs_string.c         |   34 -
 src/njs_string.h         |   34 +
 src/njs_utf8.h           |   12 +
 src/test/njs_unit_test.c |  314 ++++++++++++++++
 9 files changed, 1287 insertions(+), 34 deletions(-)

diffs (truncated from 1399 to 1000 lines):

diff -r 436d4c253d10 -r 7f5c5a425d03 auto/sources
--- a/auto/sources	Wed Jul 22 15:21:14 2020 +0300
+++ b/auto/sources	Wed Jul 22 15:21:15 2020 +0300
@@ -55,6 +55,7 @@ NJS_LIB_SRCS=" \
    src/njs_array_buffer.c \
    src/njs_typed_array.c \
    src/njs_promise.c \
+   src/njs_query_string.c \
 "
 
 NJS_LIB_TEST_SRCS=" \
diff -r 436d4c253d10 -r 7f5c5a425d03 src/njs_builtin.c
--- a/src/njs_builtin.c	Wed Jul 22 15:21:14 2020 +0300
+++ b/src/njs_builtin.c	Wed Jul 22 15:21:15 2020 +0300
@@ -50,6 +50,7 @@ static const njs_object_init_t  *njs_obj
 static const njs_object_init_t  *njs_module_init[] = {
     &njs_fs_object_init,
     &njs_crypto_object_init,
+    &njs_query_string_object_init,
     NULL
 };
 
diff -r 436d4c253d10 -r 7f5c5a425d03 src/njs_main.h
--- a/src/njs_main.h	Wed Jul 22 15:21:14 2020 +0300
+++ b/src/njs_main.h	Wed Jul 22 15:21:15 2020 +0300
@@ -79,6 +79,7 @@
 
 #include <njs_fs.h>
 #include <njs_crypto.h>
+#include <njs_query_string.h>
 
 #include <njs_event.h>
 #include <njs_module.h>
diff -r 436d4c253d10 -r 7f5c5a425d03 src/njs_query_string.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/njs_query_string.c	Wed Jul 22 15:21:15 2020 +0300
@@ -0,0 +1,912 @@
+
+/*
+ * Copyright (C) Alexander Borisov
+ * Copyright (C) Dmitry Volyntsev
+ * Copyright (C) NGINX, Inc.
+ */
+
+
+#include <njs_main.h>
+
+
+static const njs_value_t  njs_escape_str = njs_string("escape");
+static const njs_value_t  njs_unescape_str = njs_string("unescape");
+static const njs_value_t  njs_encode_uri_str =
+                                         njs_long_string("encodeURIComponent");
+static const njs_value_t  njs_decode_uri_str =
+                                         njs_long_string("decodeURIComponent");
+static const njs_value_t  njs_max_keys_str = njs_string("maxKeys");
+
+
+static njs_int_t njs_query_string_escape(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused);
+static njs_int_t njs_query_string_unescape(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_index_t unused);
+
+
+static njs_object_t *
+njs_query_string_object_alloc(njs_vm_t *vm)
+{
+    njs_object_t  *obj;
+
+    obj = njs_mp_alloc(vm->mem_pool, sizeof(njs_object_t));
+
+    if (njs_fast_path(obj != NULL)) {
+        njs_lvlhsh_init(&obj->hash);
+        njs_lvlhsh_init(&obj->shared_hash);
+        obj->type = NJS_OBJECT;
+        obj->shared = 0;
+        obj->extensible = 1;
+        obj->error_data = 0;
+        obj->fast_array = 0;
+
+        obj->__proto__ = NULL;
+        obj->slots = NULL;
+
+        return obj;
+    }
+
+    njs_memory_error(vm);
+
+    return NULL;
+}
+
+
+static njs_int_t
+njs_query_string_decode(njs_vm_t *vm, njs_value_t *value, const u_char *start,
+    size_t size)
+{
+    u_char                *dst;
+    size_t                length;
+    ssize_t               str_size;
+    uint32_t              cp;
+    njs_int_t             ret;
+    njs_chb_t             chain;
+    const u_char          *p, *end;
+    njs_unicode_decode_t  ctx;
+
+    static const int8_t  hex[256]
+        njs_aligned(32) =
+    {
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+         0,  1,  2,  3,  4,  5,  6,  7,  8,  9, -1, -1, -1, -1, -1, -1,
+        -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+    };
+
+    njs_chb_init(&chain, vm->mem_pool);
+    njs_utf8_decode_init(&ctx);
+
+    cp = 0;
+    length = 0;
+    ret = NJS_ERROR;
+
+    p = start;
+    end = p + size;
+
+    while (p < end) {
+        if (*p == '%' && end - p > 2 && hex[p[1]] >= 0 && hex[p[2]] >= 0) {
+            cp = njs_utf8_consume(&ctx, (hex[p[1]] << 4) | hex[p[2]]);
+            p += 3;
+
+        } else {
+            if (*p == '+') {
+                cp = ' ';
+                p++;
+
+            } else {
+                cp = njs_utf8_decode(&ctx, &p, end);
+            }
+        }
+
+        if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+            if (cp == NJS_UNICODE_CONTINUE) {
+                continue;
+            }
+
+            cp = NJS_UNICODE_REPLACEMENT;
+        }
+
+        dst = njs_chb_reserve(&chain, 4);
+        if (njs_slow_path(dst == NULL)) {
+            return NJS_ERROR;
+        }
+
+        njs_chb_written(&chain, njs_utf8_encode(dst, cp) - dst);
+
+        length++;
+    }
+
+    if (njs_slow_path(cp == NJS_UNICODE_CONTINUE)) {
+        dst = njs_chb_reserve(&chain, 3);
+        if (njs_slow_path(dst == NULL)) {
+            return NJS_ERROR;
+        }
+
+        njs_chb_written(&chain,
+                        njs_utf8_encode(dst, NJS_UNICODE_REPLACEMENT) - dst);
+
+        length++;
+    }
+
+    str_size = njs_chb_size(&chain);
+    if (njs_slow_path(str_size < 0)) {
+        goto failed;
+    }
+
+    dst = njs_string_alloc(vm, value, str_size, length);
+    if (njs_slow_path(dst == NULL)) {
+        goto failed;
+    }
+
+    njs_chb_join_to(&chain, dst);
+
+    ret = NJS_OK;
+
+failed:
+
+    njs_chb_destroy(&chain);
+
+    return ret;
+}
+
+
+njs_inline njs_bool_t
+njs_query_string_is_native_decoder(njs_function_t *decoder)
+{
+    return decoder->native && decoder->u.native == njs_query_string_unescape;
+}
+
+
+njs_inline njs_int_t
+njs_query_string_append(njs_vm_t *vm, njs_value_t *object, const u_char *key,
+    size_t key_size, const u_char *val, size_t val_size,
+    njs_function_t *decoder)
+{
+    uint32_t     key_length, val_length;
+    njs_int_t    ret;
+    njs_array_t  *array;
+    njs_value_t  name, value, retval;
+
+    if (njs_query_string_is_native_decoder(decoder)) {
+        ret = njs_query_string_decode(vm, &name, key, key_size);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        ret = njs_query_string_decode(vm, &value, val, val_size);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+    } else {
+
+        key_length = njs_max(njs_utf8_length(key, key_size), 0);
+        ret = njs_string_new(vm, &name, key, key_size, key_length);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        if (key_size > 0) {
+            ret = njs_function_call(vm, decoder, &njs_value_undefined, &name, 1,
+                                    &name);
+            if (njs_slow_path(ret != NJS_OK)) {
+                return ret;
+            }
+
+            if (!njs_is_string(&name)) {
+                njs_value_to_string(vm, &name, &name);
+            }
+        }
+
+        val_length = njs_max(njs_utf8_length(val, val_size), 0);
+        ret = njs_string_new(vm, &value, val, val_size, val_length);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        if (val_size > 0) {
+            ret = njs_function_call(vm, decoder, &njs_value_undefined, &value,
+                                    1, &value);
+            if (njs_slow_path(ret != NJS_OK)) {
+                return ret;
+            }
+
+            if (!njs_is_string(&value)) {
+                njs_value_to_string(vm, &value, &value);
+            }
+        }
+    }
+
+    ret = njs_value_property(vm, object, &name, &retval);
+
+    if (ret == NJS_OK) {
+        if (njs_is_array(&retval)) {
+            return njs_array_add(vm, njs_array(&retval), &value);
+        }
+
+        array = njs_array_alloc(vm, 1, 2, 0);
+        if (njs_slow_path(array == NULL)) {
+            return NJS_ERROR;
+        }
+
+        array->start[0] = retval;
+        array->start[1] = value;
+
+        njs_set_array(&value, array);
+    }
+
+    return njs_value_property_set(vm, object, &name, &value);
+}
+
+
+static u_char *
+njs_query_string_match(u_char *p, u_char *end, njs_str_t *v)
+{
+    size_t  length;
+
+    length = v->length;
+
+    if (njs_fast_path(length == 1)) {
+        p = njs_strlchr(p, end, v->start[0]);
+
+        if (p == NULL) {
+            p = end;
+        }
+
+        return p;
+    }
+
+    while (p < (end - length)) {
+        if (memcmp(p, v->start, length) == 0) {
+            return p;
+        }
+
+        p++;
+    }
+
+    return end;
+}
+
+
+static njs_int_t
+njs_query_string_parse(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
+{
+    size_t          size;
+    u_char          *end, *part, *key, *val;
+    int64_t         max_keys, count;
+    njs_int_t       ret;
+    njs_str_t       str;
+    njs_value_t     obj, value, *this, *string, *options, *arg;
+    njs_value_t     val_sep, val_eq;
+    njs_object_t    *object;
+    njs_function_t  *decode;
+
+    njs_str_t  sep = njs_str("&");
+    njs_str_t  eq = njs_str("=");
+
+    count = 0;
+    decode = NULL;
+    max_keys = 1000;
+
+    object = njs_query_string_object_alloc(vm);
+    if (njs_slow_path(object == NULL)) {
+        return NJS_ERROR;
+    }
+
+    njs_set_object(&obj, object);
+
+    this = njs_arg(args, nargs, 0);
+    string = njs_arg(args, nargs, 1);
+
+    if (njs_slow_path(!njs_is_string(string)
+                      || njs_string_length(string) == 0))
+    {
+        goto done;
+    }
+
+    njs_string_get(string, &str);
+
+    arg = njs_arg(args, nargs, 2);
+    if (!njs_is_null_or_undefined(arg)) {
+        ret = njs_value_to_string(vm, &val_sep, arg);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        if (njs_string_length(&val_sep) != 0) {
+            njs_string_get(&val_sep, &sep);
+        }
+    }
+
+    arg = njs_arg(args, nargs, 3);
+    if (!njs_is_null_or_undefined(arg)) {
+        ret = njs_value_to_string(vm, &val_eq, arg);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        if (njs_string_length(&val_eq) != 0) {
+            njs_string_get(&val_eq, &eq);
+        }
+    }
+
+    options = njs_arg(args, nargs, 4);
+
+    if (njs_is_object(options)) {
+        ret = njs_value_property(vm, options, njs_value_arg(&njs_max_keys_str),
+                                 &value);
+        if (njs_slow_path(ret == NJS_ERROR)) {
+            return ret;
+        }
+
+        ret = njs_value_to_integer(vm, &value, &max_keys);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        if (max_keys == 0) {
+            max_keys = INT64_MAX;
+        }
+
+        ret = njs_value_property(vm, options,
+                                 njs_value_arg(&njs_decode_uri_str), &value);
+
+        if (ret == NJS_OK) {
+            if (njs_slow_path(!njs_is_function(&value))) {
+                njs_type_error(vm,
+                               "option decodeURIComponent is not a function");
+                return NJS_ERROR;
+            }
+
+            decode = njs_function(&value);
+        }
+    }
+
+    if (decode == NULL) {
+        ret = njs_value_property(vm, this, njs_value_arg(&njs_unescape_str),
+                                 &value);
+
+        if (ret != NJS_OK || !njs_is_function(&value)) {
+            njs_type_error(vm, "QueryString.unescape is not a function");
+            return NJS_ERROR;
+        }
+
+        decode = njs_function(&value);
+    }
+
+    key = str.start;
+    end = str.start + str.length;
+
+    do {
+        if (count++ == max_keys) {
+            break;
+        }
+
+        part = njs_query_string_match(key, end, &sep);
+
+        if (part == key) {
+            goto next;
+        }
+
+        val = njs_query_string_match(key, end, &eq);
+
+        size = val - key;
+
+        if (val != end) {
+            val += eq.length;
+        }
+
+        ret = njs_query_string_append(vm, &obj, key, size, val, part - val,
+                                      decode);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+    next:
+
+        key = part + sep.length;
+
+    } while (key < end);
+
+done:
+
+    njs_set_object(&vm->retval, object);
+
+    return NJS_OK;
+}
+
+
+njs_inline njs_int_t
+njs_query_string_encode(njs_chb_t *chain, njs_str_t *str)
+{
+    size_t  size;
+    u_char  *p, *start, *end;
+
+    static const uint32_t  escape[] = {
+        0xffffffff,  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+
+                     /* ?>=< ;:98 7654 3210  /.-, +*)( '&%$ #"!  */
+        0xfc00987d,  /* 1111 1100 0000 0000  1001 1000 0111 1101 */
+
+                     /* _^]\ [ZYX WVUT SRQP  ONML KJIH GFED CBA@ */
+        0x78000001,  /* 0111 1000 0000 0000  0000 0000 0000 0001 */
+
+                     /*  ~}| {zyx wvut srqp  onml kjih gfed cba` */
+        0xb8000001,  /* 1011 1000 0000 0000  0000 0000 0000 0001 */
+
+        0xffffffff,  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff,  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff,  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff,  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+    };
+
+    if (chain->error) {
+        return NJS_ERROR;
+    }
+
+    if (str->length == 0) {
+        return 0;
+    }
+
+    p = str->start;
+    end = p + str->length;
+    size = str->length;
+
+    while (p < end) {
+        if (njs_need_escape(escape, *p++)) {
+            size += 2;
+        }
+    }
+
+    start = njs_chb_reserve(chain, size);
+    if (njs_slow_path(start == NULL)) {
+        return NJS_ERROR;
+    }
+
+    if (size == str->length) {
+        memcpy(start, str->start, str->length);
+        njs_chb_written(chain, str->length);
+        return str->length;
+    }
+
+    (void) njs_string_encode(escape, str->length, str->start, start);
+
+    njs_chb_written(chain, size);
+
+    return size;
+}
+
+
+njs_inline njs_bool_t
+njs_query_string_is_native_encoder(njs_function_t *encoder)
+{
+    return encoder->native && encoder->u.native == njs_query_string_escape;
+}
+
+
+njs_inline njs_int_t
+njs_query_string_encoder_call(njs_vm_t *vm, njs_chb_t *chain,
+    njs_function_t *encoder, njs_value_t *string)
+{
+    njs_str_t    str;
+    njs_int_t    ret;
+    njs_value_t  retval;
+
+    if (njs_slow_path(!njs_is_string(string))) {
+        ret = njs_value_to_string(vm, string, string);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return NJS_ERROR;
+        }
+    }
+
+    if (njs_fast_path(njs_query_string_is_native_encoder(encoder))) {
+        njs_string_get(string, &str);
+        return njs_query_string_encode(chain, &str);
+    }
+
+    ret = njs_function_call(vm, encoder, &njs_value_undefined, string, 1,
+                            &retval);
+    if (njs_slow_path(ret != NJS_OK)) {
+        return NJS_ERROR;
+    }
+
+    if (njs_slow_path(!njs_is_string(&retval))) {
+        ret = njs_value_to_string(vm, &retval, &retval);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return NJS_ERROR;
+        }
+    }
+
+    njs_string_get(&retval, &str);
+
+    ret = njs_utf8_length(str.start, str.length);
+    if (ret < 0) {
+        njs_type_error(vm, "got non-UTF8 string from encoder");
+        return NJS_ERROR;
+    }
+
+    njs_chb_append_str(chain, &str);
+
+    return ret;
+}
+
+
+njs_inline njs_int_t
+njs_query_string_push(njs_vm_t *vm, njs_chb_t *chain, njs_value_t *key,
+    njs_value_t *value, njs_string_prop_t *eq, njs_function_t *encoder)
+{
+    njs_int_t  ret, length;
+    njs_str_t  str;
+
+    length = 0;
+
+    ret = njs_query_string_encoder_call(vm, chain, encoder, key);
+    if (njs_slow_path(ret < 0)) {
+        return NJS_ERROR;
+    }
+
+    length += ret;
+
+    if (!njs_is_string(value)) {
+        ret = njs_value_to_string(vm, value, value);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return NJS_ERROR;
+        }
+    }
+
+    njs_string_get(value, &str);
+
+    if (str.length > 0) {
+        njs_chb_append(chain, eq->start, eq->size);
+        length += eq->length;
+
+        ret = njs_query_string_encoder_call(vm, chain, encoder, value);
+        if (njs_slow_path(ret < 0)) {
+            return NJS_ERROR;
+        }
+
+        length += ret;
+    }
+
+    return length;
+}
+
+
+static njs_int_t
+njs_query_string_stringify(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
+{
+    u_char             *p;
+    int64_t            len;
+    ssize_t            size;
+    uint32_t           n, i;
+    uint64_t           length;
+    njs_int_t          ret;
+    njs_chb_t          chain;
+    njs_value_t        value, retval, *string, *this, *object, *arg, *options;
+    njs_array_t        *keys, *array;
+    njs_function_t     *encode;
+    njs_string_prop_t  sep, eq;
+
+    njs_value_t  val_sep = njs_string("&");
+    njs_value_t  val_eq = njs_string("=");
+
+    (void) njs_string_prop(&sep, &val_sep);
+    (void) njs_string_prop(&eq, &val_eq);
+
+    encode = NULL;
+    this = njs_arg(args, nargs, 0);
+    object = njs_arg(args, nargs, 1);
+
+    if (njs_slow_path(!njs_is_object(object))) {
+        vm->retval = njs_string_empty;
+        return NJS_OK;
+    }
+
+    arg = njs_arg(args, nargs, 2);
+    if (!njs_is_null_or_undefined(arg)) {
+        ret = njs_value_to_string(vm, arg, arg);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        if (njs_string_length(arg) > 0) {
+            (void) njs_string_prop(&sep, arg);
+        }
+    }
+
+    arg = njs_arg(args, nargs, 3);
+    if (!njs_is_null_or_undefined(arg)) {
+        ret = njs_value_to_string(vm, arg, arg);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        if (njs_string_length(arg) > 0) {
+            (void) njs_string_prop(&eq, arg);
+        }
+    }
+
+    options = njs_arg(args, nargs, 4);
+
+    if (njs_is_object(options)) {
+        ret = njs_value_property(vm, options,
+                                 njs_value_arg(&njs_encode_uri_str), &value);
+
+        if (ret == NJS_OK) {
+            if (njs_slow_path(!njs_is_function(&value))) {
+                njs_type_error(vm,
+                               "option encodeURIComponent is not a function");
+                return NJS_ERROR;
+            }
+
+            encode = njs_function(&value);
+        }
+    }
+
+    if (encode == NULL) {
+        ret = njs_value_property(vm, this, njs_value_arg(&njs_escape_str),
+                                 &value);
+
+        if (ret != NJS_OK || !njs_is_function(&value)) {
+            njs_type_error(vm, "QueryString.escape is not a function");
+            return NJS_ERROR;
+        }
+
+        encode = njs_function(&value);
+    }
+
+    njs_chb_init(&chain, vm->mem_pool);
+
+    keys = njs_value_own_enumerate(vm, object, NJS_ENUM_KEYS, NJS_ENUM_STRING,
+                                   1);
+    if (njs_slow_path(keys == NULL)) {
+        return NJS_ERROR;
+    }
+
+    for (n = 0, length = 0; n < keys->length; n++) {
+        string = &keys->start[n];
+
+        ret = njs_value_property(vm, object, string, &value);
+        if (njs_slow_path(ret == NJS_ERROR)) {
+            goto failed;
+        }
+
+        if (njs_is_array(&value)) {
+
+            if (njs_is_fast_array(&value)) {
+                array = njs_array(&value);
+
+                for (i = 0; i < array->length; i++) {
+                    if (i != 0) {
+                        njs_chb_append(&chain, sep.start, sep.size);
+                        length += sep.length;
+                    }
+
+                    ret = njs_query_string_push(vm, &chain, string,
+                                                &array->start[i], &eq, encode);
+                    if (njs_slow_path(ret < 0)) {
+                        ret = NJS_ERROR;
+                        goto failed;
+                    }
+
+                    length += ret;
+                }
+
+                continue;
+            }
+
+            ret = njs_object_length(vm, &value, &len);
+            if (njs_slow_path(ret == NJS_ERROR)) {
+                goto failed;
+            }
+
+            for (i = 0; i < len; i++) {
+                ret = njs_value_property_i64(vm, &value, i, &retval);
+                if (njs_slow_path(ret == NJS_ERROR)) {
+                    goto failed;
+                }
+
+                if (i != 0) {
+                    njs_chb_append(&chain, sep.start, sep.size);
+                    length += sep.length;
+                }
+
+                ret = njs_query_string_push(vm, &chain, string, &retval, &eq,
+                                            encode);
+                if (njs_slow_path(ret < 0)) {
+                    ret = NJS_ERROR;
+                    goto failed;
+                }
+
+                length += ret;
+            }
+
+            continue;
+        }
+
+        if (n != 0) {
+            njs_chb_append(&chain, sep.start, sep.size);
+            length += sep.length;
+        }
+
+        ret = njs_query_string_push(vm, &chain, string, &value, &eq, encode);
+        if (njs_slow_path(ret < 0)) {
+            ret = NJS_ERROR;
+            goto failed;
+        }
+
+        length += ret;
+    }
+
+    size = njs_chb_size(&chain);
+    if (njs_slow_path(size < 0)) {
+        njs_memory_error(vm);
+        return NJS_ERROR;
+    }
+
+    p = njs_string_alloc(vm, &vm->retval, size, length);
+    if (njs_slow_path(p == NULL)) {
+        return NJS_ERROR;
+    }
+
+    njs_chb_join_to(&chain, p);
+
+    ret = NJS_OK;
+
+failed:
+
+    njs_chb_destroy(&chain);
+
+    return ret;
+}
+
+
+static njs_int_t
+njs_query_string_escape(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
+{
+    u_char       *p;
+    ssize_t      size, length;
+    njs_int_t    ret;
+    njs_str_t    str;
+    njs_chb_t    chain;
+    njs_value_t  *string, value;
+
+    string = njs_arg(args, nargs, 1);
+
+    if (!njs_is_string(string)) {
+        ret = njs_value_to_string(vm, &value, string);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        string = &value;
+    }
+
+    njs_string_get(string, &str);
+
+    njs_chb_init(&chain, vm->mem_pool);
+
+    length = njs_query_string_encode(&chain, &str);
+    if (njs_slow_path(length < 0)) {
+        return NJS_ERROR;
+    }
+
+    size = njs_chb_size(&chain);
+
+    p = njs_string_alloc(vm, &vm->retval, size, length);
+    if (njs_slow_path(p == NULL)) {
+        return NJS_ERROR;
+    }
+
+    njs_chb_join_to(&chain, p);
+
+    njs_chb_destroy(&chain);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_query_string_unescape(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
+{
+    njs_int_t    ret;
+    njs_str_t    str;
+    njs_value_t  *string, value;
+
+    string = njs_arg(args, nargs, 1);
+
+    if (!njs_is_string(string)) {
+        ret = njs_value_to_string(vm, &value, string);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        string = &value;
+    }
+
+    njs_string_get(string, &str);
+
+    return njs_query_string_decode(vm, &vm->retval, str.start, str.length);
+}
+
+
+static const njs_object_prop_t  njs_query_string_object_properties[] =
+{
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("name"),
+        .value = njs_string("querystring"),
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("parse"),
+        .value = njs_native_function(njs_query_string_parse, 4),
+        .writable = 1,
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("stringify"),
+        .value = njs_native_function(njs_query_string_stringify, 4),
+        .writable = 1,
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("escape"),
+        .value = njs_native_function(njs_query_string_escape, 1),
+        .writable = 1,
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("unescape"),
+        .value = njs_native_function(njs_query_string_unescape, 1),
+        .writable = 1,
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("decode"),
+        .value = njs_native_function(njs_query_string_parse, 4),
+        .writable = 1,
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("encode"),
+        .value = njs_native_function(njs_query_string_stringify, 4),
+        .writable = 1,
+        .configurable = 1,
+    },
+};
+
+
+const njs_object_init_t  njs_query_string_object_init = {
+    njs_query_string_object_properties,
+    njs_nitems(njs_query_string_object_properties),
+};
diff -r 436d4c253d10 -r 7f5c5a425d03 src/njs_query_string.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/njs_query_string.h	Wed Jul 22 15:21:15 2020 +0300
@@ -0,0 +1,12 @@
+
+/*
+ * Copyright (C) Alexander Borisov
+ * Copyright (C) NGINX, Inc.
+ */
+
+#ifndef _NJS_QUERY_STRING_H_INCLUDED_
+#define _NJS_QUERY_STRING_H_INCLUDED_
+
+extern const njs_object_init_t  njs_query_string_object_init;
+
+#endif /* _NJS_QUERY_STRING_H_INCLUDED_ */
diff -r 436d4c253d10 -r 7f5c5a425d03 src/njs_string.c
--- a/src/njs_string.c	Wed Jul 22 15:21:14 2020 +0300
+++ b/src/njs_string.c	Wed Jul 22 15:21:15 2020 +0300
@@ -4172,40 +4172,6 @@ const njs_object_init_t  njs_string_inst
 };
 
 
-njs_inline njs_bool_t
-njs_need_escape(const uint32_t *escape, uint32_t byte)
-{
-    return ((escape[byte >> 5] & ((uint32_t) 1 << (byte & 0x1f))) != 0);
-}
-
-
-njs_inline u_char *
-njs_string_encode(const uint32_t *escape, size_t size, const u_char *src,
-    u_char *dst)
-{
-    uint8_t              byte;
-    static const u_char  hex[16] = "0123456789ABCDEF";
-
-    do {
-        byte = *src++;
-
-        if (njs_need_escape(escape, byte)) {
-            *dst++ = '%';
-            *dst++ = hex[byte >> 4];
-            *dst++ = hex[byte & 0xf];
-
-        } else {
-            *dst++ = byte;
-        }
-
-        size--;
-

From manuel.baesler at gmail.com  Wed Jul 22 12:42:16 2020
From: manuel.baesler at gmail.com (Manuel)
Date: Wed, 22 Jul 2020 12:42:16 +0000
Subject: Nginx support for HTTP/3
In-Reply-To: <CAGmfrByqZGW+E4QsjGTZD3eFPA6o8QUmJ3fQ8j5x7R-RRYZyOQ@mail.gmail.com>
References: <CAGmfrBzhN-zJidBJTYySnMO=Uz2uhP_7U84EbUuWrZ8o6i9tjw@mail.gmail.com>
 <CAFe2+oe0PsRnxKiys0KzAg_1DKYn_K=XDk0=Bp9mnNj0k5=xxQ@mail.gmail.com>
 <CAGmfrBxRCTk_850V58XGc3MEH7hYaKY3c=A5CjO=CRF=xJBWow@mail.gmail.com>
 <DBBC16B6-87D6-4FC9-B810-254337ED2FD3@gmail.com>
 <CAGmfrByqZGW+E4QsjGTZD3eFPA6o8QUmJ3fQ8j5x7R-RRYZyOQ@mail.gmail.com>
Message-ID: <CB787393-D460-4CB5-853C-901E729149E9@gmail.com>

Hi,

you don?t have to install boring. You just have to build it and instruct the nginx build to use the boring build artifacts.

What are the error messages which commands you used?


> Am 22.07.2020 um 11:57 schrieb HARISH KUMAR Ivaturi <harishkumarivaturi at gmail.com>:
> 
> Hi Manuel
> 
> referred to https://boringssl.googlesource.com/boringssl/ for building boringssl and there is another document available as building.md . I did use but could not build boringssl . Could you elaborate me the installation steps for boringssl ? It would be helpful if you can instruct the commands.
> 
> BR
> Harish Kumar
> 
>> On Tue, Jul 21, 2020 at 10:31 PM Manuel <manuel.baesler at gmail.com> wrote:
>>  Before you can compile nginx you have to compile the boringssl libs.
>> 
>> Compiling for boringssl is documented at the boringssl webpage.
>> 
>>> Am 21.07.2020 um 16:38 schrieb HARISH KUMAR Ivaturi <harishkumarivaturi at gmail.com>:
>>> 
>>> Hi Manuel
>>> Thanks for the reply.
>>> 
>>> i did that, but i executed the command at a time , didn't do separately. The command executed was 
>>> $ ./auto/configure --with-debug --with-http_v3_module       \
>>>                        --with-cc-opt="-I../boringssl/include"   \
>>>                        --with-ld-opt="-L../boringssl/build/ssl  \
>>>                                       -L../boringssl/build/crypto"
>>> 
>>> And later says Error: OpenSSL with QUIC support is not there. 
>>> 
>>> Could you send me a video of your explanation from scratch? Building this in your linux ubuntu.
>>> 
>>> BR
>>> Harish Kumar
>>> 
>>>> On Tue, Jul 21, 2020 at 5:40 PM Manuel <manuel.baesler at gmail.com> wrote:
>>>> Hi,
>>>> 
>>>> as the readme from the hg repo indicates, you first have to build the
>>>> boring lib. (build/ssl build/crypto)
>>>> 
>>>>   $ ./auto/configure [..]
>>>> --with-ld-opt="-L../boringssl/build/ssl -L../boringssl/build/crypto"
>>>> [..]
>>>> checking for --with-ld-opt="-L../boringssl/build/ssl
>>>> -L../boringssl/build/crypto" ... found
>>>> [..]
>>>> nginx-quic/objs$ ./nginx -V
>>>> nginx version: nginx/1.19.1
>>>> built with OpenSSL 1.1.0 (compatible; BoringSSL) (running with BoringSSL)
>>>> TLS SNI support enabled
>>>> configure arguments: --with-debug --with-http_v3_module
>>>> --with-cc-opt=-I../boringssl/include
>>>> --with-ld-opt='-L../boringssl/build/ssl -L../boringssl/build/crypto'
>>>> 
>>>> Am Di., 21. Juli 2020 um 16:03 Uhr schrieb HARISH KUMAR Ivaturi
>>>> <harishkumarivaturi at gmail.com>:
>>>> >
>>>> > Hi
>>>> >
>>>> > I am Harish Kumar, Master Student at BTH Sweden. I am doing my thesis on OpenStack with HTTP/3 and dependencies to complete my thesis are Nginx Web Server with HTTP/3 feature and curl with HTTP/3. I have successfully built curl but w.r.t Nginx with HTTP/3 i could not.
>>>> >
>>>> > There are two sources which i found to build Nginx with HTTP/3 and these sources are mentioned below:
>>>> > 1) https://blog.cloudflare.com/experiment-with-http-3-using-nginx-and-quiche/
>>>> > 2) https://quic.nginx.org/readme.html
>>>> >
>>>> > I tried both and could not build nginx with HTTP/3. I get a lot of errors like openssl with QUIC support is not there though HTTP/3 is built using BoringSSL and I am not able to get a clarity. I request you to please give me some guidance and suggestions on  how to build Nginx with HTTP/3 , i followed both official sources but i failed. Unless i have a Nginx web server with HTTP/3 enabled I cannot move further with my Thesis.
>>>> >
>>>> > Please let me know.
>>>> >
>>>> > BR
>>>> > Harish Kumar
>>>> > _______________________________________________
>>>> > nginx-devel mailing list
>>>> > nginx-devel at nginx.org
>>>> > http://mailman.nginx.org/mailman/listinfo/nginx-devel
>>>> _______________________________________________
>>>> nginx-devel mailing list
>>>> nginx-devel at nginx.org
>>>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200722/de9b9b0c/attachment.htm>

From harishkumarivaturi at gmail.com  Wed Jul 22 14:27:21 2020
From: harishkumarivaturi at gmail.com (HARISH KUMAR Ivaturi)
Date: Wed, 22 Jul 2020 16:27:21 +0200
Subject: Nginx support for HTTP/3
In-Reply-To: <CB787393-D460-4CB5-853C-901E729149E9@gmail.com>
References: <CAGmfrBzhN-zJidBJTYySnMO=Uz2uhP_7U84EbUuWrZ8o6i9tjw@mail.gmail.com>
 <CAFe2+oe0PsRnxKiys0KzAg_1DKYn_K=XDk0=Bp9mnNj0k5=xxQ@mail.gmail.com>
 <CAGmfrBxRCTk_850V58XGc3MEH7hYaKY3c=A5CjO=CRF=xJBWow@mail.gmail.com>
 <DBBC16B6-87D6-4FC9-B810-254337ED2FD3@gmail.com>
 <CAGmfrByqZGW+E4QsjGTZD3eFPA6o8QUmJ3fQ8j5x7R-RRYZyOQ@mail.gmail.com>
 <CB787393-D460-4CB5-853C-901E729149E9@gmail.com>
Message-ID: <CAGmfrBxTiThnExhQiZzths+63pANCJbdnF2KSL=rUjsMrt+bCg@mail.gmail.com>

checking for PCRE library ... found
checking for PCRE JIT support ... found
checking for OpenSSL library ... found
checking for OpenSSL QUIC support ... not found

./auto/configure: error: certain modules require OpenSSL QUIC support.
You can either do not enable the modules, or install the OpenSSL library
into the system, or build the OpenSSL library statically from the source
with nginx by using --with-openssl=<path> option.


i have built openssl with ./config enable-tls1_3 --openssldir=/etc/ssl

but still it displays the same error.


On Wed, Jul 22, 2020 at 2:42 PM Manuel <manuel.baesler at gmail.com> wrote:

> Hi,
>
> you don?t have to install boring. You just have to build it and instruct
> the nginx build to use the boring build artifacts.
>
> What are the error messages which commands you used?
>
>
> Am 22.07.2020 um 11:57 schrieb HARISH KUMAR Ivaturi <
> harishkumarivaturi at gmail.com>:
>
> Hi Manuel
>
> referred to https://boringssl.googlesource.com/boringssl/ for
> building boringssl and there is another document available as building.md .
> I did use but could not build boringssl . Could you elaborate me the
> installation steps for boringssl ? It would be helpful if you can instruct
> the commands.
>
> BR
> Harish Kumar
>
> On Tue, Jul 21, 2020 at 10:31 PM Manuel <manuel.baesler at gmail.com> wrote:
>
>>  Before you can compile nginx you have to compile the boringssl libs.
>>
>> Compiling for boringssl is documented at the boringssl webpage.
>>
>> Am 21.07.2020 um 16:38 schrieb HARISH KUMAR Ivaturi <
>> harishkumarivaturi at gmail.com>:
>>
>> Hi Manuel
>> Thanks for the reply.
>>
>> i did that, but i executed the command at a time , didn't do separately.
>> The command executed was
>>
>> $ ./auto/configure --with-debug --with-http_v3_module       \
>>                        --with-cc-opt="-I../boringssl/include"   \
>>                        --with-ld-opt="-L../boringssl/build/ssl  \
>>                                       -L../boringssl/build/crypto"
>>
>>
>> And later says Error: OpenSSL with QUIC support is not there.
>>
>>
>> Could you send me a video of your explanation from scratch? Building this in your linux ubuntu.
>>
>>
>> BR
>>
>> Harish Kumar
>>
>>
>> On Tue, Jul 21, 2020 at 5:40 PM Manuel <manuel.baesler at gmail.com> wrote:
>>
>>> Hi,
>>>
>>> as the readme from the hg repo indicates, you first have to build the
>>> boring lib. (build/ssl build/crypto)
>>>
>>>   $ ./auto/configure [..]
>>> --with-ld-opt="-L../boringssl/build/ssl -L../boringssl/build/crypto"
>>> [..]
>>> checking for --with-ld-opt="-L../boringssl/build/ssl
>>> -L../boringssl/build/crypto" ... found
>>> [..]
>>> nginx-quic/objs$ ./nginx -V
>>> nginx version: nginx/1.19.1
>>> built with OpenSSL 1.1.0 (compatible; BoringSSL) (running with BoringSSL)
>>> TLS SNI support enabled
>>> configure arguments: --with-debug --with-http_v3_module
>>> --with-cc-opt=-I../boringssl/include
>>> --with-ld-opt='-L../boringssl/build/ssl -L../boringssl/build/crypto'
>>>
>>> Am Di., 21. Juli 2020 um 16:03 Uhr schrieb HARISH KUMAR Ivaturi
>>> <harishkumarivaturi at gmail.com>:
>>> >
>>> > Hi
>>> >
>>> > I am Harish Kumar, Master Student at BTH Sweden. I am doing my thesis
>>> on OpenStack with HTTP/3 and dependencies to complete my thesis are Nginx
>>> Web Server with HTTP/3 feature and curl with HTTP/3. I have successfully
>>> built curl but w.r.t Nginx with HTTP/3 i could not.
>>> >
>>> > There are two sources which i found to build Nginx with HTTP/3 and
>>> these sources are mentioned below:
>>> > 1)
>>> https://blog.cloudflare.com/experiment-with-http-3-using-nginx-and-quiche/
>>> > 2) https://quic.nginx.org/readme.html
>>> >
>>> > I tried both and could not build nginx with HTTP/3. I get a lot of
>>> errors like openssl with QUIC support is not there though HTTP/3 is built
>>> using BoringSSL and I am not able to get a clarity. I request you to please
>>> give me some guidance and suggestions on  how to build Nginx with HTTP/3 ,
>>> i followed both official sources but i failed. Unless i have a Nginx web
>>> server with HTTP/3 enabled I cannot move further with my Thesis.
>>> >
>>> > Please let me know.
>>> >
>>> > BR
>>> > Harish Kumar
>>> > _______________________________________________
>>> > nginx-devel mailing list
>>> > nginx-devel at nginx.org
>>> > http://mailman.nginx.org/mailman/listinfo/nginx-devel
>>> _______________________________________________
>>> nginx-devel mailing list
>>> nginx-devel at nginx.org
>>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>>>
>> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200722/71fb242d/attachment-0001.htm>

From sangdeuk.kwon at quantil.com  Thu Jul 23 05:29:16 2020
From: sangdeuk.kwon at quantil.com (Sangdeuk Kwon)
Date: Thu, 23 Jul 2020 14:29:16 +0900
Subject: [PATCH] During background update, nginx can't add "Range" header
Message-ID: <CAOrHi1hZPT=Sk8cORavhwaGh=58ZG9Ld4TKS=h-OPeTZVcjqHg@mail.gmail.com>

# HG changeset patch

# User Sangdeuk Kwon <sangdeuk.kwon at quantil.com>

# Date 1595481798 -32400

#      Thu Jul 23 14:23:18 2020 +0900

# Node ID 90e5ccf7c229322079ba1b61b4241ed69dfc09b2

# Parent  4f30f75dbdf33d6fae9e70086e0df5cbab7db027

During background update, nginx can't add "Range" header



If the configuration is "slice enabled" and "proxy_cache_use_stale
updating"
and "proxy_cache_background_update on",

nginx can't get "$slice_range" value during background update.

nginx sends request to upstream without "Range" header.

The re-fetched content is saved by cache key of absent "$slice_range".

So, nginx always serves stale content even though after re-fetching new
content.


slice 1k;

proxy_cache_use_stale updating;

proxy_cache_background_update on;

proxy_cache_key "$host$uri[$slice_range]



diff -r 4f30f75dbdf3 -r 90e5ccf7c229
src/http/modules/ngx_http_slice_filter_module.c
--- a/src/http/modules/ngx_http_slice_filter_module.c   Tue Jul 21 20:34:29
2020 +0300
+++ b/src/http/modules/ngx_http_slice_filter_module.c   Thu Jul 23 14:23:18
2020 +0900
@@ -400,9 +400,12 @@

     ngx_http_slice_loc_conf_t  *slcf;



     ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);

+    if (r->background && r != r->main && r->main != r->parent) {

+        ctx = ngx_http_get_module_ctx(r->parent,
ngx_http_slice_filter_module);
+    }



     if (ctx == NULL) {

-        if (r != r->main || r->headers_out.status) {

+        if ((r != r->main && !r->background) || r->headers_out.status) {

             v->not_found = 1;

             return NGX_OK;

         }
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200723/9e061cd6/attachment.htm>

From arut at nginx.com  Thu Jul 23 09:00:17 2020
From: arut at nginx.com (Roman Arutyunyan)
Date: Thu, 23 Jul 2020 09:00:17 +0000
Subject: [nginx] Xslt: disabled ranges.
Message-ID: <hg.1f3bf1734a77.1595494817.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/1f3bf1734a77
branches:  
changeset: 7686:1f3bf1734a77
user:      Roman Arutyunyan <arut at nginx.com>
date:      Wed Jul 22 22:16:19 2020 +0300
description:
Xslt: disabled ranges.

Previously, the document generated by the xslt filter was always fully sent
to client even if a range was requested and response status was 206 with
appropriate Content-Range.

The xslt module is unable to serve a range because of suspending the header
filter chain.  By the moment full response xml is buffered by the xslt filter,
range header filter is not called yet, but the range body filter has already
been called and did nothing.

The fix is to disable ranges by resetting the r->allow_ranges flag much like
the image filter that employs a similar technique.

diffstat:

 src/http/modules/ngx_http_xslt_filter_module.c |  1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diffs (11 lines):

diff -r 4f30f75dbdf3 -r 1f3bf1734a77 src/http/modules/ngx_http_xslt_filter_module.c
--- a/src/http/modules/ngx_http_xslt_filter_module.c	Tue Jul 21 20:34:29 2020 +0300
+++ b/src/http/modules/ngx_http_xslt_filter_module.c	Wed Jul 22 22:16:19 2020 +0300
@@ -233,6 +233,7 @@ ngx_http_xslt_header_filter(ngx_http_req
     ngx_http_set_ctx(r, ctx, ngx_http_xslt_filter_module);
 
     r->main_filter_need_in_memory = 1;
+    r->allow_ranges = 0;
 
     return NGX_OK;
 }

From pluknet at nginx.com  Thu Jul 23 14:43:42 2020
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Thu, 23 Jul 2020 14:43:42 +0000
Subject: [nginx] OCSP: fixed certificate reference leak.
Message-ID: <hg.d752a2c76d49.1595515422.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/d752a2c76d49
branches:  
changeset: 7687:d752a2c76d49
user:      Sergey Kandaurov <pluknet at nginx.com>
date:      Thu Jul 23 17:31:09 2020 +0300
description:
OCSP: fixed certificate reference leak.

diffstat:

 src/event/ngx_event_openssl_stapling.c |  9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diffs (67 lines):

diff -r 1f3bf1734a77 -r d752a2c76d49 src/event/ngx_event_openssl_stapling.c
--- a/src/event/ngx_event_openssl_stapling.c	Wed Jul 22 22:16:19 2020 +0300
+++ b/src/event/ngx_event_openssl_stapling.c	Thu Jul 23 17:31:09 2020 +0300
@@ -883,6 +883,7 @@ ngx_ssl_ocsp_validate(ngx_connection_t *
 
     ocsp = ngx_pcalloc(c->pool, sizeof(ngx_ssl_ocsp_t));
     if (ocsp == NULL) {
+        X509_free(cert);
         return NGX_ERROR;
     }
 
@@ -899,6 +900,7 @@ ngx_ssl_ocsp_validate(ngx_connection_t *
     if (ocsp->certs) {
         ocsp->certs = X509_chain_up_ref(ocsp->certs);
         if (ocsp->certs == NULL) {
+            X509_free(cert);
             return NGX_ERROR;
         }
     }
@@ -910,6 +912,7 @@ ngx_ssl_ocsp_validate(ngx_connection_t *
         if (store == NULL) {
             ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
                           "SSL_CTX_get_cert_store() failed");
+            X509_free(cert);
             return NGX_ERROR;
         }
 
@@ -917,6 +920,7 @@ ngx_ssl_ocsp_validate(ngx_connection_t *
         if (store_ctx == NULL) {
             ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
                           "X509_STORE_CTX_new() failed");
+            X509_free(cert);
             return NGX_ERROR;
         }
 
@@ -926,6 +930,7 @@ ngx_ssl_ocsp_validate(ngx_connection_t *
             ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
                           "X509_STORE_CTX_init() failed");
             X509_STORE_CTX_free(store_ctx);
+            X509_free(cert);
             return NGX_ERROR;
         }
 
@@ -933,6 +938,7 @@ ngx_ssl_ocsp_validate(ngx_connection_t *
         if (rc <= 0) {
             ngx_ssl_error(NGX_LOG_ERR, c->log, 0, "X509_verify_cert() failed");
             X509_STORE_CTX_free(store_ctx);
+            X509_free(cert);
             return NGX_ERROR;
         }
 
@@ -941,12 +947,15 @@ ngx_ssl_ocsp_validate(ngx_connection_t *
             ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
                           "X509_STORE_CTX_get1_chain() failed");
             X509_STORE_CTX_free(store_ctx);
+            X509_free(cert);
             return NGX_ERROR;
         }
 
         X509_STORE_CTX_free(store_ctx);
     }
 
+    X509_free(cert);
+
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
                    "ssl ocsp validate, certs:%d", sk_X509_num(ocsp->certs));
 

From harishkumarivaturi at gmail.com  Thu Jul 23 20:28:58 2020
From: harishkumarivaturi at gmail.com (HARISH KUMAR Ivaturi)
Date: Thu, 23 Jul 2020 22:28:58 +0200
Subject: Nginx support for HTTP/3
In-Reply-To: <CAGmfrBxTiThnExhQiZzths+63pANCJbdnF2KSL=rUjsMrt+bCg@mail.gmail.com>
References: <CAGmfrBzhN-zJidBJTYySnMO=Uz2uhP_7U84EbUuWrZ8o6i9tjw@mail.gmail.com>
 <CAFe2+oe0PsRnxKiys0KzAg_1DKYn_K=XDk0=Bp9mnNj0k5=xxQ@mail.gmail.com>
 <CAGmfrBxRCTk_850V58XGc3MEH7hYaKY3c=A5CjO=CRF=xJBWow@mail.gmail.com>
 <DBBC16B6-87D6-4FC9-B810-254337ED2FD3@gmail.com>
 <CAGmfrByqZGW+E4QsjGTZD3eFPA6o8QUmJ3fQ8j5x7R-RRYZyOQ@mail.gmail.com>
 <CB787393-D460-4CB5-853C-901E729149E9@gmail.com>
 <CAGmfrBxTiThnExhQiZzths+63pANCJbdnF2KSL=rUjsMrt+bCg@mail.gmail.com>
Message-ID: <CAGmfrBx1_Vp7ZcEnOKbsfoxaJkdkzERdPM8P_xGqfn46Df49SA@mail.gmail.com>

Hi Manuel

Could you reply me for the above error log.

You are just keep viewing my linkedln profile everyday ?



On Wed 22 Jul, 2020, 4:27 PM HARISH KUMAR Ivaturi, <
harishkumarivaturi at gmail.com> wrote:

> checking for PCRE library ... found
> checking for PCRE JIT support ... found
> checking for OpenSSL library ... found
> checking for OpenSSL QUIC support ... not found
>
> ./auto/configure: error: certain modules require OpenSSL QUIC support.
> You can either do not enable the modules, or install the OpenSSL library
> into the system, or build the OpenSSL library statically from the source
> with nginx by using --with-openssl=<path> option.
>
>
> i have built openssl with ./config enable-tls1_3 --openssldir=/etc/ssl
>
> but still it displays the same error.
>
>
> On Wed, Jul 22, 2020 at 2:42 PM Manuel <manuel.baesler at gmail.com> wrote:
>
>> Hi,
>>
>> you don?t have to install boring. You just have to build it and instruct
>> the nginx build to use the boring build artifacts.
>>
>> What are the error messages which commands you used?
>>
>>
>> Am 22.07.2020 um 11:57 schrieb HARISH KUMAR Ivaturi <
>> harishkumarivaturi at gmail.com>:
>>
>> Hi Manuel
>>
>> referred to https://boringssl.googlesource.com/boringssl/ for
>> building boringssl and there is another document available as building.md .
>> I did use but could not build boringssl . Could you elaborate me the
>> installation steps for boringssl ? It would be helpful if you can instruct
>> the commands.
>>
>> BR
>> Harish Kumar
>>
>> On Tue, Jul 21, 2020 at 10:31 PM Manuel <manuel.baesler at gmail.com> wrote:
>>
>>>  Before you can compile nginx you have to compile the boringssl libs.
>>>
>>> Compiling for boringssl is documented at the boringssl webpage.
>>>
>>> Am 21.07.2020 um 16:38 schrieb HARISH KUMAR Ivaturi <
>>> harishkumarivaturi at gmail.com>:
>>>
>>> Hi Manuel
>>> Thanks for the reply.
>>>
>>> i did that, but i executed the command at a time , didn't do separately.
>>> The command executed was
>>>
>>> $ ./auto/configure --with-debug --with-http_v3_module       \
>>>                        --with-cc-opt="-I../boringssl/include"   \
>>>                        --with-ld-opt="-L../boringssl/build/ssl  \
>>>                                       -L../boringssl/build/crypto"
>>>
>>>
>>> And later says Error: OpenSSL with QUIC support is not there.
>>>
>>>
>>> Could you send me a video of your explanation from scratch? Building this in your linux ubuntu.
>>>
>>>
>>> BR
>>>
>>> Harish Kumar
>>>
>>>
>>> On Tue, Jul 21, 2020 at 5:40 PM Manuel <manuel.baesler at gmail.com> wrote:
>>>
>>>> Hi,
>>>>
>>>> as the readme from the hg repo indicates, you first have to build the
>>>> boring lib. (build/ssl build/crypto)
>>>>
>>>>   $ ./auto/configure [..]
>>>> --with-ld-opt="-L../boringssl/build/ssl -L../boringssl/build/crypto"
>>>> [..]
>>>> checking for --with-ld-opt="-L../boringssl/build/ssl
>>>> -L../boringssl/build/crypto" ... found
>>>> [..]
>>>> nginx-quic/objs$ ./nginx -V
>>>> nginx version: nginx/1.19.1
>>>> built with OpenSSL 1.1.0 (compatible; BoringSSL) (running with
>>>> BoringSSL)
>>>> TLS SNI support enabled
>>>> configure arguments: --with-debug --with-http_v3_module
>>>> --with-cc-opt=-I../boringssl/include
>>>> --with-ld-opt='-L../boringssl/build/ssl -L../boringssl/build/crypto'
>>>>
>>>> Am Di., 21. Juli 2020 um 16:03 Uhr schrieb HARISH KUMAR Ivaturi
>>>> <harishkumarivaturi at gmail.com>:
>>>> >
>>>> > Hi
>>>> >
>>>> > I am Harish Kumar, Master Student at BTH Sweden. I am doing my thesis
>>>> on OpenStack with HTTP/3 and dependencies to complete my thesis are Nginx
>>>> Web Server with HTTP/3 feature and curl with HTTP/3. I have successfully
>>>> built curl but w.r.t Nginx with HTTP/3 i could not.
>>>> >
>>>> > There are two sources which i found to build Nginx with HTTP/3 and
>>>> these sources are mentioned below:
>>>> > 1)
>>>> https://blog.cloudflare.com/experiment-with-http-3-using-nginx-and-quiche/
>>>> > 2) https://quic.nginx.org/readme.html
>>>> >
>>>> > I tried both and could not build nginx with HTTP/3. I get a lot of
>>>> errors like openssl with QUIC support is not there though HTTP/3 is built
>>>> using BoringSSL and I am not able to get a clarity. I request you to please
>>>> give me some guidance and suggestions on  how to build Nginx with HTTP/3 ,
>>>> i followed both official sources but i failed. Unless i have a Nginx web
>>>> server with HTTP/3 enabled I cannot move further with my Thesis.
>>>> >
>>>> > Please let me know.
>>>> >
>>>> > BR
>>>> > Harish Kumar
>>>> > _______________________________________________
>>>> > nginx-devel mailing list
>>>> > nginx-devel at nginx.org
>>>> > http://mailman.nginx.org/mailman/listinfo/nginx-devel
>>>> _______________________________________________
>>>> nginx-devel mailing list
>>>> nginx-devel at nginx.org
>>>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>>>>
>>> _______________________________________________
>> nginx-devel mailing list
>> nginx-devel at nginx.org
>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200723/b459fcb0/attachment.htm>

From manuel.baesler at gmail.com  Thu Jul 23 22:11:23 2020
From: manuel.baesler at gmail.com (Manuel)
Date: Fri, 24 Jul 2020 00:11:23 +0200
Subject: Nginx support for HTTP/3
In-Reply-To: <CAGmfrBx1_Vp7ZcEnOKbsfoxaJkdkzERdPM8P_xGqfn46Df49SA@mail.gmail.com>
References: <CAGmfrBzhN-zJidBJTYySnMO=Uz2uhP_7U84EbUuWrZ8o6i9tjw@mail.gmail.com>
 <CAFe2+oe0PsRnxKiys0KzAg_1DKYn_K=XDk0=Bp9mnNj0k5=xxQ@mail.gmail.com>
 <CAGmfrBxRCTk_850V58XGc3MEH7hYaKY3c=A5CjO=CRF=xJBWow@mail.gmail.com>
 <DBBC16B6-87D6-4FC9-B810-254337ED2FD3@gmail.com>
 <CAGmfrByqZGW+E4QsjGTZD3eFPA6o8QUmJ3fQ8j5x7R-RRYZyOQ@mail.gmail.com>
 <CB787393-D460-4CB5-853C-901E729149E9@gmail.com>
 <CAGmfrBxTiThnExhQiZzths+63pANCJbdnF2KSL=rUjsMrt+bCg@mail.gmail.com>
 <CAGmfrBx1_Vp7ZcEnOKbsfoxaJkdkzERdPM8P_xGqfn46Df49SA@mail.gmail.com>
Message-ID: <CAFe2+ofXv8d_Z1afV0iUX1U_3iEyjZNQdOcGZV5P-Sd-k_vURA@mail.gmail.com>

Hi Harish,

I am just a stalker ;D

HTTP/3 is HTTP-over-QUIC.
Openssl does not have quic yet.
https://www.openssl.org/blog/blog/2020/02/17/QUIC-and-OpenSSL/

thats why https://hg.nginx.org/nginx-quic/file/tip/README

2. Installing

    You will need a BoringSSL [4] library that provides QUIC support

    $ hg clone -b quic https://hg.nginx.org/nginx-quic
    $ cd nginx-quic
    $ ./auto/configure --with-debug --with-http_v3_module       \
                       --with-cc-opt="-I../boringssl/include"   \
                       --with-ld-opt="-L../boringssl/build/ssl  \
                                      -L../boringssl/build/crypto"

that's why you have to build the boringssl libs before building nginx.

Best,
Manuel


Am Do., 23. Juli 2020 um 22:29 Uhr schrieb HARISH KUMAR Ivaturi
<harishkumarivaturi at gmail.com>:
>
> Hi Manuel
>
> Could you reply me for the above error log.
>
> You are just keep viewing my linkedln profile everyday
>
>
>
> On Wed 22 Jul, 2020, 4:27 PM HARISH KUMAR Ivaturi, <harishkumarivaturi at gmail.com> wrote:
>>
>> checking for PCRE library ... found
>> checking for PCRE JIT support ... found
>> checking for OpenSSL library ... found
>> checking for OpenSSL QUIC support ... not found
>>
>> ./auto/configure: error: certain modules require OpenSSL QUIC support.
>> You can either do not enable the modules, or install the OpenSSL library
>> into the system, or build the OpenSSL library statically from the source
>> with nginx by using --with-openssl=<path> option.
>>
>>
>> i have built openssl with ./config enable-tls1_3 --openssldir=/etc/ssl
>>
>> but still it displays the same error.
>>
>>
>> On Wed, Jul 22, 2020 at 2:42 PM Manuel <manuel.baesler at gmail.com> wrote:
>>>
>>> Hi,
>>>
>>> you don?t have to install boring. You just have to build it and instruct the nginx build to use the boring build artifacts.
>>>
>>> What are the error messages which commands you used?
>>>
>>>
>>> Am 22.07.2020 um 11:57 schrieb HARISH KUMAR Ivaturi <harishkumarivaturi at gmail.com>:
>>>
>>> Hi Manuel
>>>
>>> referred to https://boringssl.googlesource.com/boringssl/ for building boringssl and there is another document available as building.md . I did use but could not build boringssl . Could you elaborate me the installation steps for boringssl ? It would be helpful if you can instruct the commands.
>>>
>>> BR
>>> Harish Kumar
>>>
>>> On Tue, Jul 21, 2020 at 10:31 PM Manuel <manuel.baesler at gmail.com> wrote:
>>>>
>>>>  Before you can compile nginx you have to compile the boringssl libs.
>>>>
>>>> Compiling for boringssl is documented at the boringssl webpage.
>>>>
>>>> Am 21.07.2020 um 16:38 schrieb HARISH KUMAR Ivaturi <harishkumarivaturi at gmail.com>:
>>>>
>>>> Hi Manuel
>>>> Thanks for the reply.
>>>>
>>>> i did that, but i executed the command at a time , didn't do separately. The command executed was
>>>>
>>>> $ ./auto/configure --with-debug --with-http_v3_module       \
>>>>                        --with-cc-opt="-I../boringssl/include"   \
>>>>                        --with-ld-opt="-L../boringssl/build/ssl  \
>>>>                                       -L../boringssl/build/crypto"
>>>>
>>>>
>>>> And later says Error: OpenSSL with QUIC support is not there.
>>>>
>>>>
>>>> Could you send me a video of your explanation from scratch? Building this in your linux ubuntu.
>>>>
>>>>
>>>> BR
>>>>
>>>> Harish Kumar
>>>>
>>>>
>>>> On Tue, Jul 21, 2020 at 5:40 PM Manuel <manuel.baesler at gmail.com> wrote:
>>>>>
>>>>> Hi,
>>>>>
>>>>> as the readme from the hg repo indicates, you first have to build the
>>>>> boring lib. (build/ssl build/crypto)
>>>>>
>>>>>   $ ./auto/configure [..]
>>>>> --with-ld-opt="-L../boringssl/build/ssl -L../boringssl/build/crypto"
>>>>> [..]
>>>>> checking for --with-ld-opt="-L../boringssl/build/ssl
>>>>> -L../boringssl/build/crypto" ... found
>>>>> [..]
>>>>> nginx-quic/objs$ ./nginx -V
>>>>> nginx version: nginx/1.19.1
>>>>> built with OpenSSL 1.1.0 (compatible; BoringSSL) (running with BoringSSL)
>>>>> TLS SNI support enabled
>>>>> configure arguments: --with-debug --with-http_v3_module
>>>>> --with-cc-opt=-I../boringssl/include
>>>>> --with-ld-opt='-L../boringssl/build/ssl -L../boringssl/build/crypto'
>>>>>
>>>>> Am Di., 21. Juli 2020 um 16:03 Uhr schrieb HARISH KUMAR Ivaturi
>>>>> <harishkumarivaturi at gmail.com>:
>>>>> >
>>>>> > Hi
>>>>> >
>>>>> > I am Harish Kumar, Master Student at BTH Sweden. I am doing my thesis on OpenStack with HTTP/3 and dependencies to complete my thesis are Nginx Web Server with HTTP/3 feature and curl with HTTP/3. I have successfully built curl but w.r.t Nginx with HTTP/3 i could not.
>>>>> >
>>>>> > There are two sources which i found to build Nginx with HTTP/3 and these sources are mentioned below:
>>>>> > 1) https://blog.cloudflare.com/experiment-with-http-3-using-nginx-and-quiche/
>>>>> > 2) https://quic.nginx.org/readme.html
>>>>> >
>>>>> > I tried both and could not build nginx with HTTP/3. I get a lot of errors like openssl with QUIC support is not there though HTTP/3 is built using BoringSSL and I am not able to get a clarity. I request you to please give me some guidance and suggestions on  how to build Nginx with HTTP/3 , i followed both official sources but i failed. Unless i have a Nginx web server with HTTP/3 enabled I cannot move further with my Thesis.
>>>>> >
>>>>> > Please let me know.
>>>>> >
>>>>> > BR
>>>>> > Harish Kumar
>>>>> > _______________________________________________
>>>>> > nginx-devel mailing list
>>>>> > nginx-devel at nginx.org
>>>>> > http://mailman.nginx.org/mailman/listinfo/nginx-devel
>>>>> _______________________________________________
>>>>> nginx-devel mailing list
>>>>> nginx-devel at nginx.org
>>>>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>>>
>>> _______________________________________________
>>> nginx-devel mailing list
>>> nginx-devel at nginx.org
>>> http://mailman.nginx.org/mailman/listinfo/nginx-devel

From william.zk at antgroup.com  Fri Jul 24 03:12:48 2020
From: william.zk at antgroup.com (=?UTF-8?B?5pu+5p+vKOavheS4nSk=?=)
Date: Fri, 24 Jul 2020 11:12:48 +0800
Subject: Will nginx support quic-lb feature later?
Message-ID: <c63d1780-be5a-403f-85b0-e1807d76c621.william.zk@antgroup.com>


Dear NGINX maintainer:
     Will nginx support quic-lb feature which was described in https://tools.ietf.org/html/draft-ietf-quic-load-balancers-03 latter?  We firmly believe that nginx is the most appropriate LB product to do this(Other LB product such as LVS, have to modify kernel). If you agree with this, we are very happy to contribute our code to nginx community(We will contribute our code to nginx-quic branch).
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200724/5ecc8007/attachment.htm>

From alexander.borisov at nginx.com  Fri Jul 24 10:04:13 2020
From: alexander.borisov at nginx.com (Alexander Borisov)
Date: Fri, 24 Jul 2020 10:04:13 +0000
Subject: [njs] Fixed pre/post increment/decrement in assignment operations.
Message-ID: <hg.78f0887e5aa6.1595585053.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/78f0887e5aa6
branches:  
changeset: 1478:78f0887e5aa6
user:      Alexander Borisov <alexander.borisov at nginx.com>
date:      Thu Jul 23 13:24:10 2020 +0300
description:
Fixed pre/post increment/decrement in assignment operations.

Previously, the compound assignment operations did not create temporary
index for increments/decrements in the generator. The result was that
the increment/decrement changed the value immediately in place, which
led to incorrect calculations.

The fix is to use a separate temporary index for increments/decrements in
assignment operations.

This closes #271 issue on GitHub.

diffstat:

 src/njs_generator.c      |  30 ++++++++++++++++++++++++-
 src/njs_lexer.h          |  11 +++++----
 src/test/njs_unit_test.c |  56 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 91 insertions(+), 6 deletions(-)

diffs (165 lines):

diff -r 7f5c5a425d03 -r 78f0887e5aa6 src/njs_generator.c
--- a/src/njs_generator.c	Wed Jul 22 15:21:15 2020 +0300
+++ b/src/njs_generator.c	Thu Jul 23 13:24:10 2020 +0300
@@ -1799,7 +1799,7 @@ njs_generate_operation_assignment(njs_vm
     njs_parser_node_t *node)
 {
     njs_int_t              ret;
-    njs_index_t            index;
+    njs_index_t            index, src;
     njs_parser_node_t      *lvalue, *expr, *object, *property;
     njs_vmcode_move_t      *move;
     njs_vmcode_3addr_t     *code;
@@ -1876,6 +1876,34 @@ njs_generate_operation_assignment(njs_vm
         return ret;
     }
 
+    if (njs_slow_path(njs_parser_has_side_effect(node->right))) {
+        /*
+         * Preserve object and property values stored in variables in a case
+         * if the variables can be changed by side effects in expression.
+         */
+        if (object->token_type == NJS_TOKEN_NAME) {
+            src = object->index;
+
+            index = njs_generate_node_temp_index_get(vm, generator, object);
+            if (njs_slow_path(index == NJS_INDEX_ERROR)) {
+                return NJS_ERROR;
+            }
+
+            njs_generate_code_move(generator, move, index, src, object);
+        }
+
+        if (property->token_type == NJS_TOKEN_NAME) {
+            src = property->index;
+
+            index = njs_generate_node_temp_index_get(vm, generator, property);
+            if (njs_slow_path(index == NJS_INDEX_ERROR)) {
+                return NJS_ERROR;
+            }
+
+            njs_generate_code_move(generator, move, index, src, property);
+        }
+    }
+
     index = njs_generate_node_temp_index_get(vm, generator, node);
     if (njs_slow_path(index == NJS_INDEX_ERROR)) {
         return NJS_ERROR;
diff -r 7f5c5a425d03 -r 78f0887e5aa6 src/njs_lexer.h
--- a/src/njs_lexer.h	Wed Jul 22 15:21:15 2020 +0300
+++ b/src/njs_lexer.h	Thu Jul 23 13:24:10 2020 +0300
@@ -51,7 +51,12 @@ typedef enum {
     NJS_TOKEN_BITWISE_XOR_ASSIGNMENT,
     NJS_TOKEN_BITWISE_AND_ASSIGNMENT,
 
-#define NJS_TOKEN_LAST_ASSIGNMENT   NJS_TOKEN_BITWISE_AND_ASSIGNMENT
+    NJS_TOKEN_INCREMENT,
+    NJS_TOKEN_DECREMENT,
+    NJS_TOKEN_POST_INCREMENT,
+    NJS_TOKEN_POST_DECREMENT,
+
+#define NJS_TOKEN_LAST_ASSIGNMENT   NJS_TOKEN_POST_DECREMENT
 
     NJS_TOKEN_EQUAL,
     NJS_TOKEN_STRICT_EQUAL,
@@ -60,13 +65,9 @@ typedef enum {
 
     NJS_TOKEN_ADDITION,
     NJS_TOKEN_UNARY_PLUS,
-    NJS_TOKEN_INCREMENT,
-    NJS_TOKEN_POST_INCREMENT,
 
     NJS_TOKEN_SUBSTRACTION,
     NJS_TOKEN_UNARY_NEGATION,
-    NJS_TOKEN_DECREMENT,
-    NJS_TOKEN_POST_DECREMENT,
 
     NJS_TOKEN_MULTIPLICATION,
 
diff -r 7f5c5a425d03 -r 78f0887e5aa6 src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Wed Jul 22 15:21:15 2020 +0300
+++ b/src/test/njs_unit_test.c	Thu Jul 23 13:24:10 2020 +0300
@@ -2183,6 +2183,20 @@ static njs_unit_test_t  njs_test[] =
                  "var b = ++a; a +' '+ b"),
       njs_str("NaN NaN") },
 
+    { njs_str("var a = 0; a = a + ++a; a"),
+      njs_str("1") },
+
+    { njs_str("var a = 0; a += a + ++a; a"),
+      njs_str("1") },
+
+    { njs_str("var i = 0, arr = ['a', 'b'];"
+              "arr[i] = arr[i] + arr[++i]; arr"),
+      njs_str("ab,b") },
+
+    { njs_str("var i = 0, arr = ['a', 'b'];"
+              "arr[i] += arr[i] + arr[++i]; arr"),
+      njs_str("aab,b") },
+
     /* Post increment. */
 
     { njs_str("var a = 1;   a++"),
@@ -2265,6 +2279,20 @@ static njs_unit_test_t  njs_test[] =
                  "var b = a++; a +' '+ b"),
       njs_str("NaN NaN") },
 
+    { njs_str("var a = 0; a = a + a++; a"),
+      njs_str("0") },
+
+    { njs_str("var a = 0; a += a + a++; a"),
+      njs_str("0") },
+
+    { njs_str("var i = 1, arr = ['a', 'b'];"
+              "arr[i] = arr[i] + arr[i++]; arr"),
+      njs_str("a,bb") },
+
+    { njs_str("var i = 1, arr = ['a', 'b'];"
+              "arr[i] += arr[i] + arr[i++]; arr"),
+      njs_str("a,bbb") },
+
     /* Decrement. */
 
     { njs_str("var a = 1;   --a"),
@@ -2347,6 +2375,20 @@ static njs_unit_test_t  njs_test[] =
                  "var b = --a; a +' '+ b"),
       njs_str("NaN NaN") },
 
+    { njs_str("var a = 0; a = a + --a; a"),
+      njs_str("-1") },
+
+    { njs_str("var a = 0; a -= a + --a; a"),
+      njs_str("1") },
+
+    { njs_str("var i = 1, arr = ['a', 'b'];"
+              "arr[i] = arr[i] + arr[--i]; arr"),
+      njs_str("a,ba") },
+
+    { njs_str("var i = 1, arr = ['a', 'b'];"
+              "arr[i] += arr[i] + arr[--i]; arr"),
+      njs_str("a,bba") },
+
     /* Post decrement. */
 
     { njs_str("var a = 1;   a--"),
@@ -2429,6 +2471,20 @@ static njs_unit_test_t  njs_test[] =
                  "var b = a--; a +' '+ b"),
       njs_str("NaN NaN") },
 
+    { njs_str("var a = 0; a = a + a--; a"),
+      njs_str("0") },
+
+    { njs_str("var a = 0; a += a + a--; a"),
+      njs_str("0") },
+
+    { njs_str("var i = 1, arr = ['a', 'b'];"
+              "arr[i] = arr[i] + arr[i--]; arr"),
+      njs_str("a,bb") },
+
+    { njs_str("var i = 1, arr = ['a', 'b'];"
+              "arr[i] += arr[i] + arr[i--]; arr"),
+      njs_str("a,bbb") },
+
     /**/
 
     { njs_str("var a, b; a = 2; b = ++a + ++a; a + ' ' + b"),

From arut at nginx.com  Fri Jul 24 12:03:21 2020
From: arut at nginx.com (Roman Arutyunyan)
Date: Fri, 24 Jul 2020 15:03:21 +0300
Subject: [PATCH] During background update, nginx can't add "Range" header
In-Reply-To: <CAOrHi1hZPT=Sk8cORavhwaGh=58ZG9Ld4TKS=h-OPeTZVcjqHg@mail.gmail.com>
References: <CAOrHi1hZPT=Sk8cORavhwaGh=58ZG9Ld4TKS=h-OPeTZVcjqHg@mail.gmail.com>
Message-ID: <31A0CC58-067D-4542-BE70-934A418961DD@nginx.com>

Hi,

> On 23 Jul 2020, at 08:29, Sangdeuk Kwon <sangdeuk.kwon at quantil.com> wrote:
> 
> # HG changeset patch                                                            
> # User Sangdeuk Kwon <sangdeuk.kwon at quantil.com <mailto:sangdeuk.kwon at quantil.com>>                                
> # Date 1595481798 -32400                                                        
> #      Thu Jul 23 14:23:18 2020 +0900                                           
> # Node ID 90e5ccf7c229322079ba1b61b4241ed69dfc09b2                              
> # Parent  4f30f75dbdf33d6fae9e70086e0df5cbab7db027                              
> During background update, nginx can't add "Range" header                        
>                                                                                 
> If the configuration is "slice enabled" and "proxy_cache_use_stale updating"    
> and "proxy_cache_background_update on",                                         
> nginx can't get "$slice_range" value during background update.                  
> nginx sends request to upstream without "Range" header.                         
> The re-fetched content is saved by cache key of absent "$slice_range".          
> So, nginx always serves stale content even though after re-fetching new content.

This is correct.  The slice module does not work with background updates.

>                                                                                 
> slice 1k;                                                                       
> proxy_cache_use_stale updating;                                                 
> proxy_cache_background_update on;                                               
> proxy_cache_key "$host$uri[$slice_range]                                        
>                                                                                 
> diff -r 4f30f75dbdf3 -r 90e5ccf7c229 src/http/modules/ngx_http_slice_filter_module.c
> --- a/src/http/modules/ngx_http_slice_filter_module.c   Tue Jul 21 20:34:29 2020 +0300
> +++ b/src/http/modules/ngx_http_slice_filter_module.c   Thu Jul 23 14:23:18 2020 +0900
> @@ -400,9 +400,12 @@                                                            
>      ngx_http_slice_loc_conf_t  *slcf;                                          
>                                                                                 
>      ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);            
> +    if (r->background && r != r->main && r->main != r->parent) {               
> +        ctx = ngx_http_get_module_ctx(r->parent, ngx_http_slice_filter_module);
> +    }

Here you?re trying to get the parent request?s context to access the slice range.
But slice range is being constantly updated.  So there?s no guarantee that you get
the right slice range.

The right solution would be to pass the slice range to the background update
subrequest as an argument or a part of uri.  But that does not seem to be possible
within the current background update implementation. Probably there?s a way to save
and then access the right slice using njs.

>                                                                          
>                                                                                 
>      if (ctx == NULL) {                                                         
> -        if (r != r->main || r->headers_out.status) {                           
> +        if ((r != r->main && !r->background) || r->headers_out.status) {       
>              v->not_found = 1;                                                  
>              return NGX_OK;                                                     
>          }

Here you are creating a new ctx for the background update subrequest,
which suggests that you want to update the entire response rather that
only one slice.

> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200724/abe0d567/attachment.htm>

From xeioex at nginx.com  Fri Jul 24 16:25:20 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Fri, 24 Jul 2020 16:25:20 +0000
Subject: [njs] Improved String.prototype.toString() for ordinary strings.
Message-ID: <hg.354318f9b00f.1595607920.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/354318f9b00f
branches:  
changeset: 1479:354318f9b00f
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Fri Jul 24 14:02:04 2020 +0000
description:
Improved String.prototype.toString() for ordinary strings.

Allowing converting all strings to encodings 'hex', 'base64', 'base64url'.

diffstat:

 src/njs_string.c         |  9 ++-------
 src/test/njs_unit_test.c |  3 ---
 2 files changed, 2 insertions(+), 10 deletions(-)

diffs (40 lines):

diff -r 78f0887e5aa6 -r 354318f9b00f src/njs_string.c
--- a/src/njs_string.c	Thu Jul 23 13:24:10 2020 +0300
+++ b/src/njs_string.c	Fri Jul 24 14:02:04 2020 +0000
@@ -738,9 +738,9 @@ njs_string_prototype_value_of(njs_vm_t *
 
 
 /*
- * String.toString([encoding]).
+ * String.prototype.toString([encoding]).
  * Returns the string as is if no additional argument is provided,
- * otherwise converts a byte string into an encoded string: hex, base64,
+ * otherwise converts a string into an encoded string: hex, base64,
  * base64url.
  */
 
@@ -771,11 +771,6 @@ njs_string_prototype_to_string(njs_vm_t 
 
     (void) njs_string_prop(&string, &value);
 
-    if (njs_slow_path(string.length != 0)) {
-        njs_type_error(vm, "argument must be a byte string");
-        return NJS_ERROR;
-    }
-
     njs_string_get(&args[1], &enc);
 
     str.length = string.size;
diff -r 78f0887e5aa6 -r 354318f9b00f src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Thu Jul 23 13:24:10 2020 +0300
+++ b/src/test/njs_unit_test.c	Fri Jul 24 14:02:04 2020 +0000
@@ -895,9 +895,6 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("'A'.toString()"),
       njs_str("A") },
 
-    { njs_str("'A'.toString('hex')"),
-      njs_str("TypeError: argument must be a byte string") },
-
     { njs_str("'A'.toBytes().toString('latin1')"),
       njs_str("TypeError: Unknown encoding: \"latin1\"") },
 

From xeioex at nginx.com  Fri Jul 24 16:25:22 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Fri, 24 Jul 2020 16:25:22 +0000
Subject: [njs] Improved fs.mkdir() to support recursive directory creation.
Message-ID: <hg.cb2ff67e595d.1595607922.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/cb2ff67e595d
branches:  
changeset: 1480:cb2ff67e595d
user:      Artem S. Povalyukhin <artem.povaluhin at gmail.com>
date:      Wed Jul 15 15:51:06 2020 +0300
description:
Improved fs.mkdir() to support recursive directory creation.

diffstat:

 src/njs_fs.c               |  112 ++++++++++++++++++++++++++++++++++++++++----
 test/js/fs_promises_008.js |   79 +++++++++++++++++++++++++++++++
 test/njs_expect_test.exp   |    3 +
 3 files changed, 183 insertions(+), 11 deletions(-)

diffs (230 lines):

diff -r 354318f9b00f -r cb2ff67e595d src/njs_fs.c
--- a/src/njs_fs.c	Fri Jul 24 14:02:04 2020 +0000
+++ b/src/njs_fs.c	Wed Jul 15 15:51:06 2020 +0300
@@ -70,6 +70,9 @@ static njs_int_t njs_fs_error(njs_vm_t *
 static njs_int_t njs_fs_result(njs_vm_t *vm, njs_value_t *result,
     njs_index_t calltype, const njs_value_t* callback, njs_uint_t nargs);
 
+static njs_int_t njs_fs_make_path(njs_vm_t *vm, const char *path, mode_t md,
+    njs_bool_t recursive, njs_value_t *retval);
+
 static int njs_fs_flags(njs_vm_t *vm, njs_value_t *value, int default_flags);
 static mode_t njs_fs_mode(njs_vm_t *vm, njs_value_t *value,
     mode_t default_mode);
@@ -828,18 +831,8 @@ njs_fs_mkdir(njs_vm_t *vm, njs_value_t *
         return NJS_ERROR;
     }
 
-    if (njs_is_true(&recursive)) {
-        njs_type_error(vm, "\"options.recursive\" is not supported");
-        return NJS_ERROR;
-    }
-
-    njs_set_undefined(&retval);
-
-    ret = mkdir(file_path, md);
-    if (njs_slow_path(ret != 0)) {
-        ret = njs_fs_error(vm, "mkdir", strerror(errno), path, errno,
+    ret = njs_fs_make_path(vm, file_path, md, njs_is_true(&recursive),
                            &retval);
-    }
 
     if (ret == NJS_OK) {
         return njs_fs_result(vm, &retval, calltype, callback, 1);
@@ -1138,6 +1131,103 @@ njs_fs_fd_read(njs_vm_t *vm, int fd, njs
 }
 
 
+static njs_int_t
+njs_fs_make_path(njs_vm_t *vm, const char *path, mode_t md,
+    njs_bool_t recursive, njs_value_t *retval)
+{
+    size_t       size;
+    ssize_t      length;
+    njs_int_t    ret;
+    const char   *p, *prev;
+    njs_value_t  value;
+    struct stat  sb;
+    char         path_buf[MAXPATHLEN];
+
+    njs_set_undefined(retval);
+
+    if (!recursive) {
+        ret = mkdir(path, md);
+        if (ret != 0) {
+            goto failed;
+        }
+
+        return NJS_OK;
+    }
+
+    ret = stat(path, &sb);
+    if (ret == 0) {
+        if (!S_ISDIR(sb.st_mode)) {
+            errno = ENOTDIR;
+            goto failed;
+        }
+
+        return NJS_OK;
+    }
+
+    if (errno != ENOENT) {
+        goto failed;
+    }
+
+    p = path;
+    prev = p;
+
+    for ( ;; ) {
+        p = strchr(prev + 1, '/');
+        if (p == NULL) {
+            break;
+        }
+
+        if (njs_slow_path((p - path) > MAXPATHLEN)) {
+            njs_internal_error(vm, "too large path");
+            return NJS_OK;
+        }
+
+        memcpy(&path_buf[prev - path], &path[prev - path], p - prev);
+        path_buf[p - path] = '\0';
+
+        ret = stat(path_buf, &sb);
+        if (ret == 0) {
+            if (!S_ISDIR(sb.st_mode)) {
+                errno = ENOTDIR;
+                goto failed;
+            }
+
+        } else {
+            ret = mkdir(path_buf, md);
+            if (ret != 0) {
+                goto failed;
+            }
+        }
+
+        path_buf[p - path] = '/';
+        prev = p;
+    }
+
+    ret = mkdir(path, md);
+    if (ret != 0 && errno != EEXIST) {
+        goto failed;
+    }
+
+    return NJS_OK;
+
+failed:
+
+    size = njs_strlen(path);
+    length = njs_utf8_length((u_char *) path, size);
+    if (njs_slow_path(length < 0)) {
+        length = 0;
+    }
+
+    ret = njs_string_new(vm, &value, (u_char *) path, size, length);
+    if (ret != NJS_OK) {
+        return NJS_ERROR;
+    }
+
+    return njs_fs_error(vm, "mkdir", strerror(errno), &value, errno,
+                        retval);
+}
+
+
 static int
 njs_fs_flags(njs_vm_t *vm, njs_value_t *value, int default_flags)
 {
diff -r 354318f9b00f -r cb2ff67e595d test/js/fs_promises_008.js
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/js/fs_promises_008.js	Wed Jul 15 15:51:06 2020 +0300
@@ -0,0 +1,79 @@
+var fs = require('fs');
+var fsp  = fs.promises;
+var dname = './build/test/fs_promises_???_008/';
+var path = 'one/two/three/???';
+
+var wipePath = (root, path, nofail) => {
+    path
+        .split('/')
+        .map((x, i, a) => {
+            return root + a.slice(0, i + 1).join('/');
+        })
+        .reverse()
+        .map((dir) => {
+            try {
+                fs.rmdirSync(dir);
+            } catch (e) {
+                if (!nofail) {
+                    throw e;
+                }
+            }
+        });
+};
+
+var testSync = () => new Promise((resolve, reject) => {
+    try {
+        wipePath(dname, path + '/' + path, true);
+        fs.rmdirSync(dname);
+    } catch (e) {
+    }
+
+    try {
+        fs.mkdirSync(dname);
+
+        fs.mkdirSync(dname, { recursive: true });
+        fs.mkdirSync(dname + '/', { recursive: true });
+        fs.mkdirSync(dname + '////', { recursive: true });
+
+        fs.mkdirSync(dname + path, { recursive: true });
+        wipePath(dname, path);
+
+        fs.mkdirSync(dname + '////' + path + '////' + path + '////', { recursive: true });
+        wipePath(dname, path + '/' + path);
+
+        try {
+            fs.mkdirSync(dname + path, { recursive: true, mode: 0 });
+        } catch (e) {
+            if (e.code != 'EACCES') {
+                reject(e);
+            }
+        }
+        wipePath(dname, path, true);
+
+        try {
+            fs.mkdirSync(dname + path, { recursive: true });
+            fs.writeFileSync(dname + path + '/one', 'not dir');
+            fs.mkdirSync(dname + path + '/' + path, { recursive: true });
+        } catch (e) {
+            if (e.code != 'ENOTDIR') {
+                reject(e);
+            }
+        }
+        fs.unlinkSync(dname + path + '/one');
+        wipePath(dname, path);
+
+        fs.rmdirSync(dname);
+        resolve();
+    } catch (e) {
+        reject(e);
+    }
+});
+
+Promise.resolve()
+.then(testSync)
+.then(() => {
+    console.log('test recursive fs.mkdirSync');
+})
+.catch((e) => {
+    console.log('test failed recursive fs.mkdirSync', JSON.stringify(e));
+})
diff -r 354318f9b00f -r cb2ff67e595d test/njs_expect_test.exp
--- a/test/njs_expect_test.exp	Fri Jul 24 14:02:04 2020 +0000
+++ b/test/njs_expect_test.exp	Wed Jul 15 15:51:06 2020 +0300
@@ -1140,3 +1140,6 @@ njs_run {"./test/js/fs_promises_007.js"}
 "test fs.readdirSync
 test fs.readdir
 test fsp.readdir"
+
+njs_run {"./test/js/fs_promises_008.js"} \
+"test recursive fs.mkdirSync"

From ru at nginx.com  Fri Jul 24 19:58:04 2020
From: ru at nginx.com (Ruslan Ermilov)
Date: Fri, 24 Jul 2020 22:58:04 +0300
Subject: [PATCH] Core: enclosed parameters of macros in parentheses.
In-Reply-To: <tencent_14CA823123E68B9E69704D9CE236EDB4A309@qq.com>
References: <tencent_14CA823123E68B9E69704D9CE236EDB4A309@qq.com>
Message-ID: <20200724195804.GP49526@lo0.su>

On Sat, Jul 18, 2020 at 08:09:30PM +0800, balus wrote:
> # HG changeset patch
> # User balus <balus at foxmail.com&gt;
> # Date 1595073121 -28800
> #&nbsp; &nbsp; &nbsp; Sat Jul 18 19:52:01 2020 +0800
> # Node ID 92d9878c0c7549345f0a144cd81a6b6d45f21fc6
> # Parent&nbsp; 32a343635b50662979975e1204417bb1fc7e1b1f
> Core: enclosed parameters of macros in parentheses.

I'm going to commit the following updated patch:

# HG changeset patch
# User balus <balus at foxmail.com>
# Date 1595399139 -10800
#      Wed Jul 22 09:25:39 2020 +0300
# Node ID 827f61a45a12382773c4053e7858594c2ce611c0
# Parent  4f30f75dbdf33d6fae9e70086e0df5cbab7db027
Core: enclosed parameters of the ngx_buf.h macros in parentheses.

diff --git a/src/core/ngx_buf.h b/src/core/ngx_buf.h
--- a/src/core/ngx_buf.h
+++ b/src/core/ngx_buf.h
@@ -125,20 +125,20 @@ typedef struct {
 #define NGX_CHAIN_ERROR     (ngx_chain_t *) NGX_ERROR
 
 
-#define ngx_buf_in_memory(b)        (b->temporary || b->memory || b->mmap)
-#define ngx_buf_in_memory_only(b)   (ngx_buf_in_memory(b) && !b->in_file)
+#define ngx_buf_in_memory(b)       ((b)->temporary || (b)->memory || (b)->mmap)
+#define ngx_buf_in_memory_only(b)  (ngx_buf_in_memory(b) && !(b)->in_file)
 
 #define ngx_buf_special(b)                                                   \
-    ((b->flush || b->last_buf || b->sync)                                    \
-     && !ngx_buf_in_memory(b) && !b->in_file)
+    (((b)->flush || (b)->last_buf || (b)->sync)                              \
+     && !ngx_buf_in_memory(b) && !(b)->in_file)
 
 #define ngx_buf_sync_only(b)                                                 \
-    (b->sync                                                                 \
-     && !ngx_buf_in_memory(b) && !b->in_file && !b->flush && !b->last_buf)
+    ((b)->sync && !ngx_buf_in_memory(b)                                      \
+     && !(b)->in_file && !(b)->flush && !(b)->last_buf)
 
 #define ngx_buf_size(b)                                                      \
-    (ngx_buf_in_memory(b) ? (off_t) (b->last - b->pos):                      \
-                            (b->file_last - b->file_pos))
+    (ngx_buf_in_memory(b) ? (off_t) ((b)->last - (b)->pos):                  \
+                            ((b)->file_last - (b)->file_pos))
 
 ngx_buf_t *ngx_create_temp_buf(ngx_pool_t *pool, size_t size);
 ngx_chain_t *ngx_create_chain_of_bufs(ngx_pool_t *pool, ngx_bufs_t *bufs);
@@ -149,8 +149,8 @@ ngx_chain_t *ngx_create_chain_of_bufs(ng
 
 ngx_chain_t *ngx_alloc_chain_link(ngx_pool_t *pool);
 #define ngx_free_chain(pool, cl)                                             \
-    cl->next = pool->chain;                                                  \
-    pool->chain = cl
+    (cl)->next = (pool)->chain;                                              \
+    (pool)->chain = (cl)
 
 
 

From balus at foxmail.com  Fri Jul 24 23:50:22 2020
From: balus at foxmail.com (=?ISO-8859-1?B?YmFsdXM=?=)
Date: Sat, 25 Jul 2020 07:50:22 +0800
Subject: [PATCH] Core: enclosed parameters of macros in parentheses.
In-Reply-To: <20200724195804.GP49526@lo0.su>
References: <tencent_14CA823123E68B9E69704D9CE236EDB4A309@qq.com>
 <20200724195804.GP49526@lo0.su>
Message-ID: <tencent_C623CC94F981E7215C7EA7A94D4CEDBF3207@qq.com>

Seems more comprehensive and the commit message is more accurate
now, thank you.


------------------&nbsp;Original&nbsp;------------------
From:                                                                                                                        "nginx-devel"                                                                                    <ru at nginx.com&gt;;
Date:&nbsp;Sat, Jul 25, 2020 03:58 AM
To:&nbsp;"nginx-devel"<nginx-devel at nginx.org&gt;;

Subject:&nbsp;Re: [PATCH] Core: enclosed parameters of macros in parentheses.



On Sat, Jul 18, 2020 at 08:09:30PM +0800, balus wrote:
&gt; # HG changeset patch
&gt; # User balus <balus at foxmail.com&amp;gt;
&gt; # Date 1595073121 -28800
&gt; #&amp;nbsp; &amp;nbsp; &amp;nbsp; Sat Jul 18 19:52:01 2020 +0800
&gt; # Node ID 92d9878c0c7549345f0a144cd81a6b6d45f21fc6
&gt; # Parent&amp;nbsp; 32a343635b50662979975e1204417bb1fc7e1b1f
&gt; Core: enclosed parameters of macros in parentheses.

I'm going to commit the following updated patch:

# HG changeset patch
# User balus <balus at foxmail.com&gt;
# Date 1595399139 -10800
#&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Wed Jul 22 09:25:39 2020 +0300
# Node ID 827f61a45a12382773c4053e7858594c2ce611c0
# Parent&nbsp; 4f30f75dbdf33d6fae9e70086e0df5cbab7db027
Core: enclosed parameters of the ngx_buf.h macros in parentheses.

diff --git a/src/core/ngx_buf.h b/src/core/ngx_buf.h
--- a/src/core/ngx_buf.h
+++ b/src/core/ngx_buf.h
@@ -125,20 +125,20 @@ typedef struct {
&nbsp;#define NGX_CHAIN_ERROR&nbsp;&nbsp;&nbsp;&nbsp; (ngx_chain_t *) NGX_ERROR
&nbsp;
&nbsp;
-#define ngx_buf_in_memory(b)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; (b-&gt;temporary || b-&gt;memory || b-&gt;mmap)
-#define ngx_buf_in_memory_only(b)&nbsp;&nbsp; (ngx_buf_in_memory(b) &amp;&amp; !b-&gt;in_file)
+#define ngx_buf_in_memory(b)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ((b)-&gt;temporary || (b)-&gt;memory || (b)-&gt;mmap)
+#define ngx_buf_in_memory_only(b)&nbsp; (ngx_buf_in_memory(b) &amp;&amp; !(b)-&gt;in_file)
&nbsp;
&nbsp;#define ngx_buf_special(b)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
-&nbsp;&nbsp;&nbsp; ((b-&gt;flush || b-&gt;last_buf || b-&gt;sync)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
-&nbsp;&nbsp;&nbsp;&nbsp; &amp;&amp; !ngx_buf_in_memory(b) &amp;&amp; !b-&gt;in_file)
+&nbsp;&nbsp;&nbsp; (((b)-&gt;flush || (b)-&gt;last_buf || (b)-&gt;sync)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
+&nbsp;&nbsp;&nbsp;&nbsp; &amp;&amp; !ngx_buf_in_memory(b) &amp;&amp; !(b)-&gt;in_file)
&nbsp;
&nbsp;#define ngx_buf_sync_only(b)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
-&nbsp;&nbsp;&nbsp; (b-&gt;sync&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
-&nbsp;&nbsp;&nbsp;&nbsp; &amp;&amp; !ngx_buf_in_memory(b) &amp;&amp; !b-&gt;in_file &amp;&amp; !b-&gt;flush &amp;&amp; !b-&gt;last_buf)
+&nbsp;&nbsp;&nbsp; ((b)-&gt;sync &amp;&amp; !ngx_buf_in_memory(b)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
+&nbsp;&nbsp;&nbsp;&nbsp; &amp;&amp; !(b)-&gt;in_file &amp;&amp; !(b)-&gt;flush &amp;&amp; !(b)-&gt;last_buf)
&nbsp;
&nbsp;#define ngx_buf_size(b)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
-&nbsp;&nbsp;&nbsp; (ngx_buf_in_memory(b) ? (off_t) (b-&gt;last - b-&gt;pos):&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
-&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; (b-&gt;file_last - b-&gt;file_pos))
+&nbsp;&nbsp;&nbsp; (ngx_buf_in_memory(b) ? (off_t) ((b)-&gt;last - (b)-&gt;pos):&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ((b)-&gt;file_last - (b)-&gt;file_pos))
&nbsp;
&nbsp;ngx_buf_t *ngx_create_temp_buf(ngx_pool_t *pool, size_t size);
&nbsp;ngx_chain_t *ngx_create_chain_of_bufs(ngx_pool_t *pool, ngx_bufs_t *bufs);
@@ -149,8 +149,8 @@ ngx_chain_t *ngx_create_chain_of_bufs(ng
&nbsp;
&nbsp;ngx_chain_t *ngx_alloc_chain_link(ngx_pool_t *pool);
&nbsp;#define ngx_free_chain(pool, cl)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
-&nbsp;&nbsp;&nbsp; cl-&gt;next = pool-&gt;chain;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
-&nbsp;&nbsp;&nbsp; pool-&gt;chain = cl
+&nbsp;&nbsp;&nbsp; (cl)-&gt;next = (pool)-&gt;chain;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
+&nbsp;&nbsp;&nbsp; (pool)-&gt;chain = (cl)
&nbsp;
&nbsp;
&nbsp;
_______________________________________________
nginx-devel mailing list
nginx-devel at nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200725/2b693fcb/attachment.htm>

From sangdeuk.kwon at quantil.com  Mon Jul 27 05:10:32 2020
From: sangdeuk.kwon at quantil.com (Sangdeuk Kwon)
Date: Mon, 27 Jul 2020 14:10:32 +0900
Subject: [PATCH] During background update, nginx can't add "Range" header
Message-ID: <CAOrHi1jCT4ioE9Jufk7DA4MHhWTUfjKvk2duxmp_1uhKV3CH0w@mail.gmail.com>

> Hi,

Thank you for your explanation.

I added comments. Please check it one more time.

>>* On 23 Jul 2020, at 08:29, Sangdeuk Kwon <sangdeuk.kwon at quantil.com <http://mailman.nginx.org/mailman/listinfo/nginx-devel>> wrote:
*>> >>* # HG changeset patch
>*>* # User Sangdeuk Kwon <sangdeuk.kwon at quantil.com <http://mailman.nginx.org/mailman/listinfo/nginx-devel> <mailto:sangdeuk.kwon at quantil.com <http://mailman.nginx.org/mailman/listinfo/nginx-devel>>>
*>>* # Date 1595481798 -32400
>*>* #      Thu Jul 23 14:23:18 2020 +0900
>*>* # Node ID 90e5ccf7c229322079ba1b61b4241ed69dfc09b2
>*>* # Parent  4f30f75dbdf33d6fae9e70086e0df5cbab7db027
>*>* During background update, nginx can't add "Range" header
>*>*
>*>* If the configuration is "slice enabled" and "proxy_cache_use_stale updating"
*>>* and "proxy_cache_background_update on",
>*>* nginx can't get "$slice_range" value during background update.
>*>* nginx sends request to upstream without "Range" header.
>*>* The re-fetched content is saved by cache key of absent "$slice_range".
*>>* So, nginx always serves stale content even though after
re-fetching new content.
*>
> This is correct.  The slice module does not work with background updates.
>
>>*
>*>* slice 1k;
>*>* proxy_cache_use_stale updating;
>*>* proxy_cache_background_update on;
>*>* proxy_cache_key "$host$uri[$slice_range]
>*>*
>*>* diff -r 4f30f75dbdf3 -r 90e5ccf7c229 src/http/modules/ngx_http_slice_filter_module.c
*>>* --- a/src/http/modules/ngx_http_slice_filter_module.c   Tue Jul
21 20:34:29 2020 +0300
*>>* +++ b/src/http/modules/ngx_http_slice_filter_module.c   Thu Jul
23 14:23:18 2020 +0900
*>>* @@ -400,9 +400,12 @@
*>>*      ngx_http_slice_loc_conf_t  *slcf;
*>>
             >>*      ctx = ngx_http_get_module_ctx(r,
ngx_http_slice_filter_module);
*>>* +    if (r->background && r != r->main && r->main != r->parent) {
*>>* +        ctx = ngx_http_get_module_ctx(r->parent,
ngx_http_slice_filter_module);
*>>* +    }
*>
> Here you?re trying to get the parent request?s context to access the slice range.
> But slice range is being constantly updated.  So there?s no guarantee that you get
> the right slice range.

> The right solution would be to pass the slice range to the background update
> subrequest as an argument or a part of uri.  But that does not seem to be possible
> within the current background update implementation. Probably there?s a way to save
> and then access the right slice using njs.
>

The posted requests are saved as below order in r->main->posted_requests.
Subrequest of background update is executed next to same slice request
before next slice request.
Background update's r->parent refers same slice request.
So, r->parent's ngx_http_slice_ctx_t has correct "Range" value for
background update.

r: slice #1
r->main->posted_requests: -> background update request (slice #1) ->
next slice request (slice #2)

r: background update request (slice #1) (r->main=slice #1, r->parent=slice #1)
r->main->posted_requests: -> next slice request (slice #2)

r: slice #2
r->main->posted_requests: -> background update request (slice #2) ->
next slice request (slice #3)

r: background update request (slice #2) (r->main=slice #1, r->parent=slice #2)
r->main->posted_requests: -> next slice request (slice #3)

r: slice #3 (if no need background update)
r->main->posted_requests: -> next slice request (slice #4)

r: slice #4
r->main->posted_requests: -> background update request (slice #4) ->
next slice request (slice #5)

r: background update request (slice #4) (r->main=slice #1, r->parent=slice #4)
r->main->posted_requests: -> next slice request (slice #5)



But, in slice #1 case, "Range" value of ctx is already updated to next
value (slice #2) when it's created.
So, background update of slice #1 can't use r->parent's ctx.

When user request has "Range" header(only one slice case), background
update can generate "Range" value by "Range" header.

"r->main == r->parent" condition is for slice #1 and user request has
"Range" header(only one slice case)
"r->main != r->parent" condition is for slice #2, slice #3, and ...

>>                                                                          >>                                                                                 >>*      if (ctx == NULL) {
*>>* -        if (r != r->main || r->headers_out.status) {
*>>* +        if ((r != r->main && !r->background) ||
r->headers_out.status) {
*>>*              v->not_found = 1;
*>>*              return NGX_OK;
*>>*          }
*>
> Here you are creating a new ctx for the background update subrequest,
> which suggests that you want to update the entire response rather that
> only one slice.
>

It's created when slice #1 and user request has "Range" header(only
one slice case)
>>* _______________________________________________
*>>* nginx-devel mailing list
*>>* nginx-devel at nginx.org
<http://mailman.nginx.org/mailman/listinfo/nginx-devel>
*>>* http://mailman.nginx.org/mailman/listinfo/nginx-devel
<http://mailman.nginx.org/mailman/listinfo/nginx-devel>*
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200727/76c2547e/attachment.htm>

From ru at nginx.com  Mon Jul 27 10:27:57 2020
From: ru at nginx.com (Ruslan Ermilov)
Date: Mon, 27 Jul 2020 10:27:57 +0000
Subject: [nginx] Core: enclosed parameters of the ngx_buf.h macros in
 parentheses.
Message-ID: <hg.2f9214713666.1595845677.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/2f9214713666
branches:  
changeset: 7688:2f9214713666
user:      balus <balus at foxmail.com>
date:      Mon Jul 27 13:21:51 2020 +0300
description:
Core: enclosed parameters of the ngx_buf.h macros in parentheses.

diffstat:

 src/core/ngx_buf.h |  20 ++++++++++----------
 1 files changed, 10 insertions(+), 10 deletions(-)

diffs (43 lines):

diff -r d752a2c76d49 -r 2f9214713666 src/core/ngx_buf.h
--- a/src/core/ngx_buf.h	Thu Jul 23 17:31:09 2020 +0300
+++ b/src/core/ngx_buf.h	Mon Jul 27 13:21:51 2020 +0300
@@ -125,20 +125,20 @@ typedef struct {
 #define NGX_CHAIN_ERROR     (ngx_chain_t *) NGX_ERROR
 
 
-#define ngx_buf_in_memory(b)        (b->temporary || b->memory || b->mmap)
-#define ngx_buf_in_memory_only(b)   (ngx_buf_in_memory(b) && !b->in_file)
+#define ngx_buf_in_memory(b)       ((b)->temporary || (b)->memory || (b)->mmap)
+#define ngx_buf_in_memory_only(b)  (ngx_buf_in_memory(b) && !(b)->in_file)
 
 #define ngx_buf_special(b)                                                   \
-    ((b->flush || b->last_buf || b->sync)                                    \
-     && !ngx_buf_in_memory(b) && !b->in_file)
+    (((b)->flush || (b)->last_buf || (b)->sync)                              \
+     && !ngx_buf_in_memory(b) && !(b)->in_file)
 
 #define ngx_buf_sync_only(b)                                                 \
-    (b->sync                                                                 \
-     && !ngx_buf_in_memory(b) && !b->in_file && !b->flush && !b->last_buf)
+    ((b)->sync && !ngx_buf_in_memory(b)                                      \
+     && !(b)->in_file && !(b)->flush && !(b)->last_buf)
 
 #define ngx_buf_size(b)                                                      \
-    (ngx_buf_in_memory(b) ? (off_t) (b->last - b->pos):                      \
-                            (b->file_last - b->file_pos))
+    (ngx_buf_in_memory(b) ? (off_t) ((b)->last - (b)->pos):                  \
+                            ((b)->file_last - (b)->file_pos))
 
 ngx_buf_t *ngx_create_temp_buf(ngx_pool_t *pool, size_t size);
 ngx_chain_t *ngx_create_chain_of_bufs(ngx_pool_t *pool, ngx_bufs_t *bufs);
@@ -149,8 +149,8 @@ ngx_chain_t *ngx_create_chain_of_bufs(ng
 
 ngx_chain_t *ngx_alloc_chain_link(ngx_pool_t *pool);
 #define ngx_free_chain(pool, cl)                                             \
-    cl->next = pool->chain;                                                  \
-    pool->chain = cl
+    (cl)->next = (pool)->chain;                                              \
+    (pool)->chain = (cl)
 
 
 

From xeioex at nginx.com  Mon Jul 27 14:36:20 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Mon, 27 Jul 2020 14:36:20 +0000
Subject: [njs] Fixed TOCTOU in fs.mkdir() introduced in cb2ff67e595d.
Message-ID: <hg.6078d0c735b4.1595860580.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/6078d0c735b4
branches:  
changeset: 1481:6078d0c735b4
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Mon Jul 27 14:18:15 2020 +0000
description:
Fixed TOCTOU in fs.mkdir() introduced in cb2ff67e595d.

Found by Coverity (CID 1465508).

diffstat:

 src/njs_fs.c |  75 +++++++++++++++++++++++++++++------------------------------
 1 files changed, 37 insertions(+), 38 deletions(-)

diffs (137 lines):

diff -r cb2ff67e595d -r 6078d0c735b4 src/njs_fs.c
--- a/src/njs_fs.c	Wed Jul 15 15:51:06 2020 +0300
+++ b/src/njs_fs.c	Mon Jul 27 14:18:15 2020 +0000
@@ -1135,96 +1135,95 @@ static njs_int_t
 njs_fs_make_path(njs_vm_t *vm, const char *path, mode_t md,
     njs_bool_t recursive, njs_value_t *retval)
 {
-    size_t       size;
+    int          err;
     ssize_t      length;
     njs_int_t    ret;
-    const char   *p, *prev;
+    const char   *p, *prev, *end;
     njs_value_t  value;
     struct stat  sb;
     char         path_buf[MAXPATHLEN];
 
     njs_set_undefined(retval);
 
+    end = path + njs_strlen(path);
+
     if (!recursive) {
         ret = mkdir(path, md);
         if (ret != 0) {
+            err = errno;
             goto failed;
         }
 
         return NJS_OK;
     }
 
-    ret = stat(path, &sb);
-    if (ret == 0) {
-        if (!S_ISDIR(sb.st_mode)) {
-            errno = ENOTDIR;
-            goto failed;
-        }
-
-        return NJS_OK;
-    }
-
-    if (errno != ENOENT) {
-        goto failed;
-    }
-
     p = path;
     prev = p;
 
     for ( ;; ) {
         p = strchr(prev + 1, '/');
         if (p == NULL) {
-            break;
+            p = end;
         }
 
         if (njs_slow_path((p - path) > MAXPATHLEN)) {
             njs_internal_error(vm, "too large path");
-            return NJS_OK;
+            return NJS_ERROR;
         }
 
         memcpy(&path_buf[prev - path], &path[prev - path], p - prev);
         path_buf[p - path] = '\0';
 
-        ret = stat(path_buf, &sb);
-        if (ret == 0) {
-            if (!S_ISDIR(sb.st_mode)) {
-                errno = ENOTDIR;
-                goto failed;
+        ret = mkdir(path_buf, md);
+        err = errno;
+
+        switch (ret) {
+        case 0:
+            break;
+
+        case EACCES:
+        case ENOTDIR:
+        case EPERM:
+            goto failed;
+
+        case EEXIST:
+        default:
+            ret = stat(path_buf, &sb);
+            if (ret == 0) {
+                if (!S_ISDIR(sb.st_mode)) {
+                    err = ENOTDIR;
+                    goto failed;
+                }
+
+                break;
             }
 
-        } else {
-            ret = mkdir(path_buf, md);
-            if (ret != 0) {
-                goto failed;
-            }
+            goto failed;
+        }
+
+        if (p == end) {
+            break;
         }
 
         path_buf[p - path] = '/';
         prev = p;
     }
 
-    ret = mkdir(path, md);
-    if (ret != 0 && errno != EEXIST) {
-        goto failed;
-    }
-
     return NJS_OK;
 
 failed:
 
-    size = njs_strlen(path);
-    length = njs_utf8_length((u_char *) path, size);
+    length = njs_utf8_length((u_char *) path, end - path);
     if (njs_slow_path(length < 0)) {
         length = 0;
     }
 
-    ret = njs_string_new(vm, &value, (u_char *) path, size, length);
+    ret = njs_string_new(vm, &value, (u_char *) path, end - path, length);
     if (ret != NJS_OK) {
         return NJS_ERROR;
     }
 
-    return njs_fs_error(vm, "mkdir", strerror(errno), &value, errno,
-                        retval);
+    return njs_fs_error(vm, "mkdir", strerror(err), &value, err, retval);
 }
 
 

From xeioex at nginx.com  Mon Jul 27 14:36:22 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Mon, 27 Jul 2020 14:36:22 +0000
Subject: [njs] Improved readability of surrogate pairs handling.
Message-ID: <hg.d4c69313ac6c.1595860582.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/d4c69313ac6c
branches:  
changeset: 1482:d4c69313ac6c
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Mon Jul 27 14:34:35 2020 +0000
description:
Improved readability of surrogate pairs handling.

diffstat:

 src/njs_json.c    |   2 +-
 src/njs_parser.c  |   4 ++--
 src/njs_string.c  |   4 ++--
 src/njs_string.h  |  10 ----------
 src/njs_unicode.h |  12 ++++++++++++
 src/njs_utf16.c   |  11 ++++-------
 6 files changed, 21 insertions(+), 22 deletions(-)

diffs (123 lines):

diff -r 6078d0c735b4 -r d4c69313ac6c src/njs_json.c
--- a/src/njs_json.c	Mon Jul 27 14:18:15 2020 +0000
+++ b/src/njs_json.c	Mon Jul 27 14:34:35 2020 +0000
@@ -738,7 +738,7 @@ njs_json_parse_string(njs_json_parse_ctx
                 p += 4;
 
                 if (njs_fast_path(njs_surrogate_trailing(utf_low))) {
-                    utf = njs_string_surrogate_pair(utf, utf_low);
+                    utf = njs_surrogate_pair(utf, utf_low);
 
                 } else if (njs_surrogate_leading(utf_low)) {
                     utf = NJS_UNICODE_REPLACEMENT;
diff -r 6078d0c735b4 -r d4c69313ac6c src/njs_parser.c
--- a/src/njs_parser.c	Mon Jul 27 14:18:15 2020 +0000
+++ b/src/njs_parser.c	Mon Jul 27 14:34:35 2020 +0000
@@ -8088,7 +8088,7 @@ njs_parser_escape_string_create(njs_pars
 
         if (cp_pair != 0) {
             if (njs_fast_path(njs_surrogate_trailing(cp))) {
-                cp = njs_string_surrogate_pair(cp_pair, cp);
+                cp = njs_surrogate_pair(cp_pair, cp);
 
             } else if (njs_slow_path(njs_surrogate_leading(cp))) {
                 cp = NJS_UNICODE_REPLACEMENT;
@@ -8238,7 +8238,7 @@ njs_parser_escape_string_calc_length(njs
 
         if (cp_pair != 0) {
             if (njs_fast_path(njs_surrogate_trailing(cp))) {
-                cp = njs_string_surrogate_pair(cp_pair, cp);
+                cp = njs_surrogate_pair(cp_pair, cp);
 
             } else if (njs_slow_path(njs_surrogate_leading(cp))) {
                 cp = NJS_UNICODE_REPLACEMENT;
diff -r 6078d0c735b4 -r d4c69313ac6c src/njs_string.c
--- a/src/njs_string.c	Mon Jul 27 14:18:15 2020 +0000
+++ b/src/njs_string.c	Mon Jul 27 14:34:35 2020 +0000
@@ -4272,7 +4272,7 @@ njs_string_encode_uri(njs_vm_t *vm, njs_
                         goto uri_error;
                     }
 
-                    cp = njs_string_surrogate_pair(cp, cp_low);
+                    cp = njs_surrogate_pair(cp, cp_low);
                     size += njs_utf8_size(cp) * 3;
                     continue;
                 }
@@ -4312,7 +4312,7 @@ njs_string_encode_uri(njs_vm_t *vm, njs_
 
         if (njs_slow_path(njs_surrogate_leading(cp))) {
             cp_low = njs_utf8_decode(&ctx, &src, end);
-            cp = njs_string_surrogate_pair(cp, cp_low);
+            cp = njs_surrogate_pair(cp, cp_low);
         }
 
         njs_utf8_encode(encode, cp);
diff -r 6078d0c735b4 -r d4c69313ac6c src/njs_string.h
--- a/src/njs_string.h	Mon Jul 27 14:18:15 2020 +0000
+++ b/src/njs_string.h	Mon Jul 27 14:34:35 2020 +0000
@@ -26,16 +26,6 @@
 /* The maximum signed int32_t. */
 #define NJS_STRING_MAX_LENGTH  0x7fffffff
 
-#define njs_surrogate_leading(cp)    ((cp) >= 0xd800 && (cp) <= 0xdbff)
-
-#define njs_surrogate_trailing(cp)   ((cp) >= 0xdc00 && (cp) <= 0xdfff)
-
-#define njs_surrogate_any(cp)        ((cp) >= 0xd800 && (cp) <= 0xdfff)
-
-/* Converting surrogate pair to code point.  */
-#define njs_string_surrogate_pair(high, low)                                  \
-    (0x10000 + ((high - 0xd800) << 10) + (low - 0xdc00))
-
 /*
  * NJS_STRING_MAP_STRIDE should be power of two to use shift and binary
  * AND operations instead of division and remainder operations but no
diff -r 6078d0c735b4 -r d4c69313ac6c src/njs_unicode.h
--- a/src/njs_unicode.h	Mon Jul 27 14:18:15 2020 +0000
+++ b/src/njs_unicode.h	Mon Jul 27 14:34:35 2020 +0000
@@ -23,5 +23,17 @@ typedef struct {
     u_char    upper;
 } njs_unicode_decode_t;
 
+#define njs_surrogate_leading(cp)                                             \
+    (((unsigned) (cp) - 0xd800) <= 0xdbff - 0xd800)
+
+#define njs_surrogate_trailing(cp)                                            \
+    (((unsigned) (cp) - 0xdc00) <= 0xdfff - 0xdc00)
+
+#define njs_surrogate_any(cp)                                                 \
+    (((unsigned) (cp) - 0xd800) <= 0xdfff - 0xd800)
+
+#define njs_surrogate_pair(high, low)                                         \
+    (0x10000 + (((high) - 0xd800) << 10) + ((low) - 0xdc00))
+
 
 #endif /* _NJS_UNICODE_H_INCLUDED_ */
diff -r 6078d0c735b4 -r d4c69313ac6c src/njs_utf16.c
--- a/src/njs_utf16.c	Mon Jul 27 14:18:15 2020 +0000
+++ b/src/njs_utf16.c	Mon Jul 27 14:34:35 2020 +0000
@@ -79,9 +79,8 @@ lead_state:
 #endif
 
     if (ctx->codepoint != 0x00) {
-        if ((unsigned) (unit - 0xDC00) <= (0xDFFF - 0xDC00)) {
-            unit = 0x10000 + ((ctx->codepoint - 0xD800) << 10)
-                   + (unit - 0xDC00);
+        if (njs_surrogate_trailing(unit)) {
+            unit = njs_surrogate_pair(ctx->codepoint, unit);
 
             ctx->codepoint = 0x00;
 
@@ -96,10 +95,8 @@ lead_state:
         return NJS_UNICODE_ERROR;
     }
 
-    /* Surrogate pair. */
-
-    if ((unsigned) (unit - 0xD800) <= (0xDFFF - 0xD800)) {
-        if ((unsigned) (unit - 0xDC00) <= (0xDFFF - 0xDC00)) {
+    if (njs_surrogate_any(unit)) {
+        if (njs_surrogate_trailing(unit)) {
             return NJS_UNICODE_ERROR;
         }
 

From mdounin at mdounin.ru  Mon Jul 27 18:18:01 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 27 Jul 2020 18:18:01 +0000
Subject: [nginx] FastCGI: fixed zero size buf alerts on extra data (ticket
 #2018).
Message-ID: <hg.da8d758aabeb.1595873881.6026610855610030274@dev.nginx>

details:   https://hg.nginx.org/nginx/rev/da8d758aabeb
branches:  
changeset: 7689:da8d758aabeb
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Mon Jul 27 16:02:15 2020 +0300
description:
FastCGI: fixed zero size buf alerts on extra data (ticket #2018).

After 05e42236e95b (1.19.1) responses with extra data might result in
zero size buffers being generated and "zero size buf" alerts in writer
(if f->rest happened to be 0 when processing additional stdout data).

diffstat:

 src/http/modules/ngx_http_fastcgi_module.c |  28 ++++++++++++++++++++++------
 1 files changed, 22 insertions(+), 6 deletions(-)

diffs (59 lines):

diff -r 2f9214713666 -r da8d758aabeb src/http/modules/ngx_http_fastcgi_module.c
--- a/src/http/modules/ngx_http_fastcgi_module.c	Mon Jul 27 13:21:51 2020 +0300
+++ b/src/http/modules/ngx_http_fastcgi_module.c	Mon Jul 27 16:02:15 2020 +0300
@@ -2306,6 +2306,18 @@ ngx_http_fastcgi_input_filter(ngx_event_
             break;
         }
 
+        if (f->rest == -2) {
+            f->rest = r->upstream->headers_in.content_length_n;
+        }
+
+        if (f->rest == 0) {
+            ngx_log_error(NGX_LOG_WARN, p->log, 0,
+                          "upstream sent more data than specified in "
+                          "\"Content-Length\" header");
+            p->upstream_done = 1;
+            break;
+        }
+
         cl = ngx_chain_get_free_buf(p->pool, &p->free);
         if (cl == NULL) {
             return NGX_ERROR;
@@ -2349,11 +2361,7 @@ ngx_http_fastcgi_input_filter(ngx_event_
             b->last = f->last;
         }
 
-        if (f->rest == -2) {
-            f->rest = r->upstream->headers_in.content_length_n;
-        }
-
-        if (f->rest >= 0) {
+        if (f->rest > 0) {
 
             if (b->last - b->pos > f->rest) {
                 ngx_log_error(NGX_LOG_WARN, p->log, 0,
@@ -2564,6 +2572,14 @@ ngx_http_fastcgi_non_buffered_filter(voi
             break;
         }
 
+        if (f->rest == 0) {
+            ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
+                          "upstream sent more data than specified in "
+                          "\"Content-Length\" header");
+            u->length = 0;
+            break;
+        }
+
         cl = ngx_chain_get_free_buf(r->pool, &u->free_bufs);
         if (cl == NULL) {
             return NGX_ERROR;
@@ -2594,7 +2610,7 @@ ngx_http_fastcgi_non_buffered_filter(voi
             b->last = f->last;
         }
 
-        if (f->rest >= 0) {
+        if (f->rest > 0) {
 
             if (b->last - b->pos > f->rest) {
                 ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,

From rohitm at chelsio.com  Wed Jul 29 09:13:26 2020
From: rohitm at chelsio.com (Rohit Maheshwari)
Date: Wed, 29 Jul 2020 14:43:26 +0530
Subject: [PATCH] Enable TCP offload support on tls connecitons
Message-ID: <32c7a0088f6d259163bb.1596014006@redhouse-blr-asicdesigners-com>

# HG changeset patch
# User Rohit Maheshwari <rohitm at chelsio.com>
# Date 1595354862 -19800
#      Tue Jul 21 23:37:42 2020 +0530
# Node ID 32c7a0088f6d259163bb2820db0b44d36659b333
# Parent  32a343635b50662979975e1204417bb1fc7e1b1f
Enable TCP offload support on tls connecitons

Linux provides feasibility to enable TOE BYPASS iff setsockopt
of type TCP_ULP is called just after socket creation. After that
only, driver can register its TCP callbacks and move to TCP
listen.

diff -r 32a343635b50 -r 32c7a0088f6d src/core/ngx_connection.c
--- a/src/core/ngx_connection.c	Thu Jul 09 16:21:37 2020 +0300
+++ b/src/core/ngx_connection.c	Tue Jul 21 23:37:42 2020 +0530
@@ -495,6 +495,10 @@ ngx_open_listening_sockets(ngx_cycle_t *
                 return NGX_ERROR;
             }
 
+#if (NGX_LINUX && NGX_TOE)
+	    setsockopt(s, SOL_TCP, TCP_ULP, "tls", sizeof("tls"));
+#endif
+
             if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR,
                            (const void *) &reuseaddr, sizeof(int))
                 == -1)
diff -r 32a343635b50 -r 32c7a0088f6d src/core/ngx_resolver.c
--- a/src/core/ngx_resolver.c	Thu Jul 09 16:21:37 2020 +0300
+++ b/src/core/ngx_resolver.c	Tue Jul 21 23:37:42 2020 +0530
@@ -4502,6 +4502,10 @@ ngx_tcp_connect(ngx_resolver_connection_
         return NGX_ERROR;
     }
 
+#if (NGX_LINUX && NGX_TOE)
+    setsockopt(s, SOL_TCP, TCP_ULP, "tls", sizeof("tls"));
+#endif
+
     c = ngx_get_connection(s, &rec->log);
 
     if (c == NULL) {

From rohitm at chelsio.com  Wed Jul 29 09:14:06 2020
From: rohitm at chelsio.com (Rohit Maheshwari)
Date: Wed, 29 Jul 2020 14:44:06 +0530
Subject: [PATCH] Enable TCP offload support on tls connecitons
Message-ID: <32c7a0088f6d259163bb.1596014046@redhouse-blr-asicdesigners-com>

# HG changeset patch
# User Rohit Maheshwari <rohitm at chelsio.com>
# Date 1595354862 -19800
#      Tue Jul 21 23:37:42 2020 +0530
# Node ID 32c7a0088f6d259163bb2820db0b44d36659b333
# Parent  32a343635b50662979975e1204417bb1fc7e1b1f
Enable TCP offload support on tls connecitons

Linux provides feasibility to enable TOE BYPASS iff setsockopt
of type TCP_ULP is called just after socket creation. After that
only, driver can register its TCP callbacks and move to TCP
listen.

diff -r 32a343635b50 -r 32c7a0088f6d src/core/ngx_connection.c
--- a/src/core/ngx_connection.c	Thu Jul 09 16:21:37 2020 +0300
+++ b/src/core/ngx_connection.c	Tue Jul 21 23:37:42 2020 +0530
@@ -495,6 +495,10 @@ ngx_open_listening_sockets(ngx_cycle_t *
                 return NGX_ERROR;
             }
 
+#if (NGX_LINUX && NGX_TOE)
+	    setsockopt(s, SOL_TCP, TCP_ULP, "tls", sizeof("tls"));
+#endif
+
             if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR,
                            (const void *) &reuseaddr, sizeof(int))
                 == -1)
diff -r 32a343635b50 -r 32c7a0088f6d src/core/ngx_resolver.c
--- a/src/core/ngx_resolver.c	Thu Jul 09 16:21:37 2020 +0300
+++ b/src/core/ngx_resolver.c	Tue Jul 21 23:37:42 2020 +0530
@@ -4502,6 +4502,10 @@ ngx_tcp_connect(ngx_resolver_connection_
         return NGX_ERROR;
     }
 
+#if (NGX_LINUX && NGX_TOE)
+    setsockopt(s, SOL_TCP, TCP_ULP, "tls", sizeof("tls"));
+#endif
+
     c = ngx_get_connection(s, &rec->log);
 
     if (c == NULL) {

From mat999 at gmail.com  Wed Jul 29 13:37:43 2020
From: mat999 at gmail.com (Mathew Heard)
Date: Wed, 29 Jul 2020 23:37:43 +1000
Subject: Trac #915 - "Upgrade" header should not be proxied over h2
Message-ID: <CAE5sJtQ_UZpYS1scFVuwmNqLTu+D=fW4F3sdAi7oN9rqvuNTaA@mail.gmail.com>

Hi All,

If anyone else is searching for a better solution to this bug (perhaps in
Apache) the following nginx patch works for me
https://github.com/splitice/nginx/commit/a91fdb43793f006bda06d980a89fd1dfb428ebee

Tested on 3 different ios devices and an Apache h2 backend.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20200729/3b69e951/attachment.htm>

From mdounin at mdounin.ru  Wed Jul 29 14:18:15 2020
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Wed, 29 Jul 2020 17:18:15 +0300
Subject: [PATCH] Enable TCP offload support on tls connecitons
In-Reply-To: <32c7a0088f6d259163bb.1596014046@redhouse-blr-asicdesigners-com>
References: <32c7a0088f6d259163bb.1596014046@redhouse-blr-asicdesigners-com>
Message-ID: <20200729141815.GF12747@mdounin.ru>

Hello!

On Wed, Jul 29, 2020 at 02:44:06PM +0530, Rohit Maheshwari wrote:

> # HG changeset patch
> # User Rohit Maheshwari <rohitm at chelsio.com>
> # Date 1595354862 -19800
> #      Tue Jul 21 23:37:42 2020 +0530
> # Node ID 32c7a0088f6d259163bb2820db0b44d36659b333
> # Parent  32a343635b50662979975e1204417bb1fc7e1b1f
> Enable TCP offload support on tls connecitons
> 
> Linux provides feasibility to enable TOE BYPASS iff setsockopt
> of type TCP_ULP is called just after socket creation. After that
> only, driver can register its TCP callbacks and move to TCP
> listen.

For TLS connections, setsockopt(TCP_ULP, "tls") is expected to be 
called by the SSL layer.  You may want to elaborate more on why 
you are trying to call it on all connections instead.

> 
> diff -r 32a343635b50 -r 32c7a0088f6d src/core/ngx_connection.c
> --- a/src/core/ngx_connection.c	Thu Jul 09 16:21:37 2020 +0300
> +++ b/src/core/ngx_connection.c	Tue Jul 21 23:37:42 2020 +0530
> @@ -495,6 +495,10 @@ ngx_open_listening_sockets(ngx_cycle_t *
>                  return NGX_ERROR;
>              }
>  
> +#if (NGX_LINUX && NGX_TOE)
> +	    setsockopt(s, SOL_TCP, TCP_ULP, "tls", sizeof("tls"));
> +#endif
> +
>              if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR,
>                             (const void *) &reuseaddr, sizeof(int))
>                  == -1)
> diff -r 32a343635b50 -r 32c7a0088f6d src/core/ngx_resolver.c
> --- a/src/core/ngx_resolver.c	Thu Jul 09 16:21:37 2020 +0300
> +++ b/src/core/ngx_resolver.c	Tue Jul 21 23:37:42 2020 +0530
> @@ -4502,6 +4502,10 @@ ngx_tcp_connect(ngx_resolver_connection_
>          return NGX_ERROR;
>      }
>  
> +#if (NGX_LINUX && NGX_TOE)
> +    setsockopt(s, SOL_TCP, TCP_ULP, "tls", sizeof("tls"));
> +#endif
> +
>      c = ngx_get_connection(s, &rec->log);
>  
>      if (c == NULL) {
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

-- 
Maxim Dounin
http://mdounin.ru/

From xeioex at nginx.com  Wed Jul 29 17:01:22 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Wed, 29 Jul 2020 17:01:22 +0000
Subject: [njs] Fixed njs_prop_handler_t inline documentation.
Message-ID: <hg.cb490ee06ac2.1596042082.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/cb490ee06ac2
branches:  
changeset: 1483:cb490ee06ac2
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Wed Jul 29 17:00:23 2020 +0000
description:
Fixed njs_prop_handler_t inline documentation.

diffstat:

 src/njs.h |  6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

diffs (16 lines):

diff -r d4c69313ac6c -r cb490ee06ac2 src/njs.h
--- a/src/njs.h	Mon Jul 27 14:34:35 2020 +0000
+++ b/src/njs.h	Wed Jul 29 17:00:23 2020 +0000
@@ -73,9 +73,9 @@ extern const njs_value_t            njs_
  *
  * njs_prop_handler_t is expected to return:
  *   NJS_OK - handler executed successfully;
- *   NJS_ERROR - some error, vm->retval contains appropriate exception;
- *   NJS_DECLINED - handler was applied to inappropriate object, vm->retval
- *   contains undefined value.
+ *   NJS_DECLINED - handler was applied to inappropriate object, retval
+ *   contains undefined value;
+ *   NJS_ERROR - some error, vm->retval contains appropriate exception.
  */
 typedef njs_int_t (*njs_prop_handler_t) (njs_vm_t *vm, njs_object_prop_t *prop,
     njs_value_t *value, njs_value_t *setval, njs_value_t *retval);

From xeioex at nginx.com  Thu Jul 30 17:48:16 2020
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 30 Jul 2020 17:48:16 +0000
Subject: [njs] Introduced tags for NJS_DATA type.
Message-ID: <hg.0fad09ddb37a.1596131296.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/0fad09ddb37a
branches:  
changeset: 1484:0fad09ddb37a
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu Jul 30 17:47:05 2020 +0000
description:
Introduced tags for NJS_DATA type.

NJS_DATA is designed to contain arbitrary opaque pointers.
Tags are used to distinguish different opaque pointers.

diffstat:

 src/njs.h                |   2 -
 src/njs_crypto.c         |  60 ++++++++++++++++++++---------------------------
 src/njs_extern.c         |  11 +++-----
 src/njs_promise.c        |  14 +++++-----
 src/njs_value.c          |  14 -----------
 src/njs_value.h          |  25 +++++++++++++++++--
 src/test/njs_unit_test.c |   4 +++
 7 files changed, 63 insertions(+), 67 deletions(-)

diffs (396 lines):

diff -r cb490ee06ac2 -r 0fad09ddb37a src/njs.h
--- a/src/njs.h	Wed Jul 29 17:00:23 2020 +0000
+++ b/src/njs.h	Thu Jul 30 17:47:05 2020 +0000
@@ -341,11 +341,9 @@ NJS_EXPORT void njs_vm_memory_error(njs_
 NJS_EXPORT void njs_value_undefined_set(njs_value_t *value);
 NJS_EXPORT void njs_value_boolean_set(njs_value_t *value, int yn);
 NJS_EXPORT void njs_value_number_set(njs_value_t *value, double num);
-NJS_EXPORT void njs_value_data_set(njs_value_t *value, void *data);
 
 NJS_EXPORT uint8_t njs_value_bool(const njs_value_t *value);
 NJS_EXPORT double njs_value_number(const njs_value_t *value);
-NJS_EXPORT void *njs_value_data(const njs_value_t *value);
 NJS_EXPORT njs_function_t *njs_value_function(const njs_value_t *value);
 
 NJS_EXPORT uint16_t njs_vm_prop_magic16(njs_object_prop_t *prop);
diff -r cb490ee06ac2 -r 0fad09ddb37a src/njs_crypto.c
--- a/src/njs_crypto.c	Wed Jul 29 17:00:23 2020 +0000
+++ b/src/njs_crypto.c	Thu Jul 30 17:47:05 2020 +0000
@@ -184,7 +184,7 @@ njs_crypto_create_hash(njs_vm_t *vm, njs
 
     alg->init(&dgst->u);
 
-    njs_set_data(&hash->value, dgst);
+    njs_set_data(&hash->value, dgst, NJS_DATA_TAG_CRYPTO_HASH);
     njs_set_object_value(&vm->retval, hash);
 
     return NJS_OK;
@@ -196,6 +196,7 @@ njs_hash_prototype_update(njs_vm_t *vm, 
     njs_index_t unused)
 {
     njs_str_t     data;
+    njs_value_t   *this;
     njs_digest_t  *dgst;
 
     if (njs_slow_path(nargs < 2 || !njs_is_string(&args[1]))) {
@@ -203,19 +204,16 @@ njs_hash_prototype_update(njs_vm_t *vm, 
         return NJS_ERROR;
     }
 
-    if (njs_slow_path(!njs_is_object_value(&args[0]))) {
-        njs_type_error(vm, "\"this\" is not an object_value");
-        return NJS_ERROR;
-    }
+    this = njs_argument(args, 0);
 
-    if (njs_slow_path(!njs_is_data(njs_object_value(&args[0])))) {
-        njs_type_error(vm, "value of \"this\" is not a data type");
+    if (njs_slow_path(!njs_is_object_data(this, NJS_DATA_TAG_CRYPTO_HASH))) {
+        njs_type_error(vm, "\"this\" is not a hash object");
         return NJS_ERROR;
     }
 
     njs_string_get(&args[1], &data);
 
-    dgst = njs_value_data(njs_object_value(&args[0]));
+    dgst = njs_object_data(this);
 
     if (njs_slow_path(dgst->alg == NULL)) {
         njs_error(vm, "Digest already called");
@@ -224,7 +222,7 @@ njs_hash_prototype_update(njs_vm_t *vm, 
 
     dgst->alg->update(&dgst->u, data.start, data.length);
 
-    vm->retval = args[0];
+    vm->retval = *this;
 
     return NJS_OK;
 }
@@ -237,6 +235,7 @@ njs_hash_prototype_digest(njs_vm_t *vm, 
     u_char            digest[32], *p;
     njs_int_t         ret;
     njs_str_t         enc_name, str;
+    njs_value_t       *this;
     njs_digest_t      *dgst;
     njs_hash_alg_t    *alg;
     njs_crypto_enc_t  *enc;
@@ -246,13 +245,10 @@ njs_hash_prototype_digest(njs_vm_t *vm, 
         return NJS_ERROR;
     }
 
-    if (njs_slow_path(!njs_is_object_value(&args[0]))) {
-        njs_type_error(vm, "\"this\" is not an object_value");
-        return NJS_ERROR;
-    }
+    this = njs_argument(args, 0);
 
-    if (njs_slow_path(!njs_is_data(njs_object_value(&args[0])))) {
-        njs_type_error(vm, "value of \"this\" is not a data type");
+    if (njs_slow_path(!njs_is_object_data(this, NJS_DATA_TAG_CRYPTO_HASH))) {
+        njs_type_error(vm, "\"this\" is not a hash object");
         return NJS_ERROR;
     }
 
@@ -267,7 +263,7 @@ njs_hash_prototype_digest(njs_vm_t *vm, 
         }
     }
 
-    dgst = njs_value_data(njs_object_value(&args[0]));
+    dgst = njs_object_data(this);
 
     if (njs_slow_path(dgst->alg == NULL)) {
         njs_error(vm, "Digest already called");
@@ -465,7 +461,7 @@ njs_crypto_create_hmac(njs_vm_t *vm, njs
         return NJS_ERROR;
     }
 
-    njs_set_data(&hmac->value, ctx);
+    njs_set_data(&hmac->value, ctx, NJS_DATA_TAG_CRYPTO_HMAC);
     njs_set_object_value(&vm->retval, hmac);
 
     return NJS_OK;
@@ -476,27 +472,25 @@ static njs_int_t
 njs_hmac_prototype_update(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     njs_index_t unused)
 {
-    njs_str_t   data;
-    njs_hmac_t  *ctx;
+    njs_str_t    data;
+    njs_hmac_t   *ctx;
+    njs_value_t  *this;
 
     if (njs_slow_path(nargs < 2 || !njs_is_string(&args[1]))) {
         njs_type_error(vm, "data must be a string");
         return NJS_ERROR;
     }
 
-    if (njs_slow_path(!njs_is_object_value(&args[0]))) {
-        njs_type_error(vm, "\"this\" is not an object_value");
-        return NJS_ERROR;
-    }
+    this = njs_argument(args, 0);
 
-    if (njs_slow_path(!njs_is_data(njs_object_value(&args[0])))) {
-        njs_type_error(vm, "value of \"this\" is not a data type");
+    if (njs_slow_path(!njs_is_object_data(this, NJS_DATA_TAG_CRYPTO_HMAC))) {
+        njs_type_error(vm, "\"this\" is not a hash object");
         return NJS_ERROR;
     }
 
     njs_string_get(&args[1], &data);
 
-    ctx = njs_value_data(njs_object_value(&args[0]));
+    ctx = njs_object_data(this);
 
     if (njs_slow_path(ctx->alg == NULL)) {
         njs_error(vm, "Digest already called");
@@ -505,7 +499,7 @@ njs_hmac_prototype_update(njs_vm_t *vm, 
 
     ctx->alg->update(&ctx->u, data.start, data.length);
 
-    vm->retval = args[0];
+    vm->retval = *this;
 
     return NJS_OK;
 }
@@ -519,6 +513,7 @@ njs_hmac_prototype_digest(njs_vm_t *vm, 
     njs_str_t         enc_name, str;
     njs_int_t         ret;
     njs_hmac_t        *ctx;
+    njs_value_t       *this;
     njs_hash_alg_t    *alg;
     njs_crypto_enc_t  *enc;
 
@@ -527,13 +522,10 @@ njs_hmac_prototype_digest(njs_vm_t *vm, 
         return NJS_ERROR;
     }
 
-    if (njs_slow_path(!njs_is_object_value(&args[0]))) {
-        njs_type_error(vm, "\"this\" is not an object_value");
-        return NJS_ERROR;
-    }
+    this = njs_argument(args, 0);
 
-    if (njs_slow_path(!njs_is_data(njs_object_value(&args[0])))) {
-        njs_type_error(vm, "value of \"this\" is not a data type");
+    if (njs_slow_path(!njs_is_object_data(this, NJS_DATA_TAG_CRYPTO_HMAC))) {
+        njs_type_error(vm, "\"this\" is not a hash object");
         return NJS_ERROR;
     }
 
@@ -548,7 +540,7 @@ njs_hmac_prototype_digest(njs_vm_t *vm, 
         }
     }
 
-    ctx = njs_value_data(njs_object_value(&args[0]));
+    ctx = njs_object_data(this);
 
     if (njs_slow_path(ctx->alg == NULL)) {
         njs_error(vm, "Digest already called");
diff -r cb490ee06ac2 -r 0fad09ddb37a src/njs_extern.c
--- a/src/njs_extern.c	Wed Jul 29 17:00:23 2020 +0000
+++ b/src/njs_extern.c	Thu Jul 30 17:47:05 2020 +0000
@@ -202,7 +202,7 @@ njs_external_prop_handler(njs_vm_t *vm, 
         ov->object.__proto__ = &vm->prototypes[NJS_OBJ_TYPE_OBJECT].object;
         ov->object.slots = slots;
 
-        njs_set_data(&ov->value, external);
+        njs_set_data(&ov->value, external, NJS_DATA_TAG_EXTERNAL);
         njs_set_object_value(retval, ov);
     }
 
@@ -313,7 +313,7 @@ njs_vm_external_create(njs_vm_t *vm, njs
     ov->object.slots = slots;
 
     njs_set_object_value(value, ov);
-    njs_set_data(&ov->value, external);
+    njs_set_data(&ov->value, external, NJS_DATA_TAG_EXTERNAL);
 
     return NJS_OK;
 }
@@ -322,11 +322,8 @@ njs_vm_external_create(njs_vm_t *vm, njs
 njs_external_ptr_t
 njs_vm_external(njs_vm_t *vm, const njs_value_t *value)
 {
-    if (njs_fast_path(njs_is_object_value(value))) {
-        value = njs_object_value(value);
-        if (njs_fast_path(njs_is_data(value))) {
-            return njs_value_data(value);
-        }
+    if (njs_fast_path(njs_is_object_data(value, NJS_DATA_TAG_EXTERNAL))) {
+        return njs_object_data(value);
     }
 
     return NULL;
diff -r cb490ee06ac2 -r 0fad09ddb37a src/njs_promise.c
--- a/src/njs_promise.c	Wed Jul 29 17:00:23 2020 +0000
+++ b/src/njs_promise.c	Thu Jul 30 17:47:05 2020 +0000
@@ -101,7 +101,7 @@ njs_promise_alloc(njs_vm_t *vm)
     njs_queue_init(&data->reject_queue);
 
     njs_set_promise(&vm->retval, promise);
-    njs_value_data_set(&promise->value, data);
+    njs_set_data(&promise->value, data, 0);
 
     return promise;
 
@@ -453,7 +453,7 @@ njs_promise_trigger_reactions(njs_vm_t *
         function = njs_promise_create_function(vm);
         function->u.native = njs_promise_reaction_job;
 
-        njs_set_data(&arguments[0], reaction);
+        njs_set_data(&arguments[0], reaction, 0);
         arguments[1] = *value;
 
         ret = njs_promise_add_event(vm, function, arguments, 2);
@@ -472,7 +472,7 @@ njs_promise_fulfill(njs_vm_t *vm, njs_pr
     njs_queue_t         queue;
     njs_promise_data_t  *data;
 
-    data = njs_value_data(&promise->value);
+    data = njs_data(&promise->value);
 
     data->result = *value;
     data->state = NJS_PROMISE_FULFILL;
@@ -500,7 +500,7 @@ njs_promise_reject(njs_vm_t *vm, njs_pro
     njs_queue_t         queue;
     njs_promise_data_t  *data;
 
-    data = njs_value_data(&promise->value);
+    data = njs_data(&promise->value);
 
     data->result = *reason;
     data->state = NJS_PROMISE_REJECTED;
@@ -845,7 +845,7 @@ njs_promise_perform_then(njs_vm_t *vm, n
     }
 
     promise = njs_promise(value);
-    data = njs_value_data(&promise->value);
+    data = njs_data(&promise->value);
 
     fulfilled_reaction = njs_mp_alloc(vm->mem_pool,
                                       sizeof(njs_promise_reaction_t));
@@ -878,12 +878,12 @@ njs_promise_perform_then(njs_vm_t *vm, n
         function->u.native = njs_promise_reaction_job;
 
         if (data->state == NJS_PROMISE_REJECTED) {
-            njs_set_data(&arguments[0], rejected_reaction);
+            njs_set_data(&arguments[0], rejected_reaction, 0);
 
             /* TODO: HostPromiseRejectionTracker */
 
         } else {
-            njs_set_data(&arguments[0], fulfilled_reaction);
+            njs_set_data(&arguments[0], fulfilled_reaction, 0);
         }
 
         arguments[1] = data->result;
diff -r cb490ee06ac2 -r 0fad09ddb37a src/njs_value.c
--- a/src/njs_value.c	Wed Jul 29 17:00:23 2020 +0000
+++ b/src/njs_value.c	Thu Jul 30 17:47:05 2020 +0000
@@ -371,13 +371,6 @@ njs_value_number_set(njs_value_t *value,
 }
 
 
-void
-njs_value_data_set(njs_value_t *value, void *data)
-{
-    njs_set_data(value, data);
-}
-
-
 uint8_t
 njs_value_bool(const njs_value_t *value)
 {
@@ -392,13 +385,6 @@ njs_value_number(const njs_value_t *valu
 }
 
 
-void *
-njs_value_data(const njs_value_t *value)
-{
-    return njs_data(value);
-}
-
-
 njs_function_t *
 njs_value_function(const njs_value_t *value)
 {
diff -r cb490ee06ac2 -r 0fad09ddb37a src/njs_value.h
--- a/src/njs_value.h	Wed Jul 29 17:00:23 2020 +0000
+++ b/src/njs_value.h	Thu Jul 30 17:47:05 2020 +0000
@@ -76,6 +76,15 @@ typedef enum {
 } njs_value_type_t;
 
 
+typedef enum {
+    NJS_DATA_TAG_ANY = 0,
+    NJS_DATA_TAG_EXTERNAL,
+    NJS_DATA_TAG_CRYPTO_HASH,
+    NJS_DATA_TAG_CRYPTO_HMAC,
+    NJS_DATA_TAG_MAX
+} njs_data_tag_t;
+
+
 typedef struct njs_string_s           njs_string_t;
 typedef struct njs_object_s           njs_object_t;
 typedef struct njs_object_value_s     njs_object_value_t;
@@ -599,8 +608,8 @@ typedef struct {
     ((value)->type <= NJS_STRING)
 
 
-#define njs_is_data(value)                                                    \
-    ((value)->type == NJS_DATA)
+#define njs_is_data(value, tag)                                               \
+    ((value)->type == NJS_DATA && value->data.magic32 == (tag))
 
 
 #define njs_is_object(value)                                                  \
@@ -616,6 +625,11 @@ typedef struct {
     ((value)->type == NJS_OBJECT_VALUE)
 
 
+#define njs_is_object_data(_value, tag)                                       \
+    (((_value)->type == NJS_OBJECT_VALUE)                                     \
+     && njs_is_data(njs_object_value(_value), tag))
+
+
 #define njs_is_object_string(value)                                           \
     ((value)->type == NJS_OBJECT_STRING)
 
@@ -748,6 +762,10 @@ typedef struct {
     (&(_value)->data.u.object_value->value)
 
 
+#define njs_object_data(_value)                                               \
+    njs_data(njs_object_value(_value))
+
+
 #define njs_set_undefined(value)                                              \
     *(value) = njs_value_undefined
 
@@ -852,8 +870,9 @@ njs_set_symbol(njs_value_t *value, uint3
 
 
 njs_inline void
-njs_set_data(njs_value_t *value, void *data)
+njs_set_data(njs_value_t *value, void *data, njs_data_tag_t tag)
 {
+    value->data.magic32 = tag;
     value->data.u.data = data;
     value->type = NJS_DATA;
     value->data.truth = 1;
diff -r cb490ee06ac2 -r 0fad09ddb37a src/test/njs_unit_test.c
--- a/src/test/njs_unit_test.c	Wed Jul 29 17:00:23 2020 +0000
+++ b/src/test/njs_unit_test.c	Thu Jul 30 17:47:05 2020 +0000
@@ -16956,6 +16956,10 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("typeof require('crypto').createHmac('md5', 'a')"),
       njs_str("object") },
 
+    { njs_str("var cr = require('crypto'); var h = cr.createHash('sha1');"
+              "h.update.call(cr.createHmac('sha1', 's'), '')"),
+      njs_str("TypeError: \"this\" is not a hash object") },
+
     /* setTimeout(). */
 
     { njs_str("setTimeout()"),

From alexander.borisov at nginx.com  Thu Jul 30 18:34:52 2020
From: alexander.borisov at nginx.com (Alexander Borisov)
Date: Thu, 30 Jul 2020 18:34:52 +0000
Subject: [njs] Introduced TextEncoder/TextDecoder implementation.
Message-ID: <hg.bcd1a41c6a67.1596134092.5965299922797593991@dev.nginx>

details:   https://hg.nginx.org/njs/rev/bcd1a41c6a67
branches:  
changeset: 1485:bcd1a41c6a67
user:      Alexander Borisov <alexander.borisov at nginx.com>
date:      Tue Jul 28 16:58:59 2020 +0300
description:
Introduced TextEncoder/TextDecoder implementation.

According to WHATWG encoding spec.

diffstat:

 auto/sources             |    1 +
 src/njs_builtin.c        |   22 +
 src/njs_encoding.c       |  804 +++++++++++++++++++++++++++++++++++++++++++++++
 src/njs_encoding.h       |   14 +
 src/njs_main.h           |    1 +
 src/njs_object_hash.h    |   30 +
 src/njs_typed_array.c    |   73 ++-
 src/njs_typed_array.h    |    2 +
 src/njs_unicode.h        |    1 +
 src/njs_value.h          |    7 +
 src/njs_vm.h             |    4 +
 src/test/njs_unit_test.c |  143 ++++++++
 12 files changed, 1072 insertions(+), 30 deletions(-)

diffs (truncated from 1336 to 1000 lines):

diff -r 0fad09ddb37a -r bcd1a41c6a67 auto/sources
--- a/auto/sources	Thu Jul 30 17:47:05 2020 +0000
+++ b/auto/sources	Tue Jul 28 16:58:59 2020 +0300
@@ -56,6 +56,7 @@ NJS_LIB_SRCS=" \
    src/njs_typed_array.c \
    src/njs_promise.c \
    src/njs_query_string.c \
+   src/njs_encoding.c \
 "
 
 NJS_LIB_TEST_SRCS=" \
diff -r 0fad09ddb37a -r bcd1a41c6a67 src/njs_builtin.c
--- a/src/njs_builtin.c	Thu Jul 30 17:47:05 2020 +0000
+++ b/src/njs_builtin.c	Tue Jul 28 16:58:59 2020 +0300
@@ -71,6 +71,8 @@ static const njs_object_type_init_t *con
     &njs_date_type_init,
     &njs_promise_type_init,
     &njs_array_buffer_type_init,
+    &njs_text_decoder_type_init,
+    &njs_text_encoder_type_init,
 
     /* Hidden types. */
 
@@ -1283,6 +1285,26 @@ static const njs_object_prop_t  njs_glob
 
     {
         .type = NJS_PROPERTY_HANDLER,
+        .name = njs_string("TextDecoder"),
+        .value = njs_prop_handler2(njs_top_level_constructor,
+                                   NJS_OBJ_TYPE_TEXT_DECODER,
+                                   NJS_TEXT_DECODER_HASH),
+        .writable = 1,
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY_HANDLER,
+        .name = njs_string("TextEncoder"),
+        .value = njs_prop_handler2(njs_top_level_constructor,
+                                   NJS_OBJ_TYPE_TEXT_ENCODER,
+                                   NJS_TEXT_ENCODER_HASH),
+        .writable = 1,
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY_HANDLER,
         .name = njs_string("Uint8Array"),
         .value = njs_prop_handler2(njs_top_level_constructor,
                                    NJS_OBJ_TYPE_UINT8_ARRAY,
diff -r 0fad09ddb37a -r bcd1a41c6a67 src/njs_encoding.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/njs_encoding.c	Tue Jul 28 16:58:59 2020 +0300
@@ -0,0 +1,804 @@
+
+/*
+ * Copyright (C) Alexander Borisov
+ * Copyright (C) NGINX, Inc.
+ */
+
+
+#include <njs_main.h>
+
+
+typedef enum {
+    NJS_ENCODING_UTF8,
+} njs_encoding_t;
+
+
+typedef struct {
+    njs_encoding_t        encoding;
+    njs_bool_t            fatal;
+    njs_bool_t            ignore_bom;
+
+    uint32_t              codepoint;
+    njs_unicode_decode_t  ctx;
+} njs_encoding_decode_t;
+
+
+typedef struct {
+    njs_str_t             name;
+    njs_encoding_t        encoding;
+} njs_encoding_label_t;
+
+
+static njs_encoding_label_t  njs_encoding_labels[] =
+{
+    { njs_str("utf-8"), NJS_ENCODING_UTF8 },
+    { njs_str("utf8") , NJS_ENCODING_UTF8 },
+    { njs_null_str, 0 }
+};
+
+
+static njs_int_t njs_text_encoder_encode_utf8(njs_vm_t *vm,
+    njs_string_prop_t *prop);
+static njs_int_t njs_text_decoder_arg_encoding(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_encoding_decode_t *data);
+static njs_int_t njs_text_decoder_arg_options(njs_vm_t *vm, njs_value_t *args,
+    njs_uint_t nargs, njs_encoding_decode_t *data);
+
+
+static njs_int_t
+njs_text_encoder_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
+{
+    njs_object_t        *proto;
+    njs_object_value_t  *ov;
+
+    if (!vm->top_frame->ctor) {
+        njs_type_error(vm, "Constructor of TextEncoder requires 'new'");
+        return NJS_ERROR;
+    }
+
+    ov = njs_mp_alloc(vm->mem_pool, sizeof(njs_object_value_t));
+    if (njs_slow_path(ov == NULL)) {
+        njs_memory_error(vm);
+        return NJS_ERROR;
+    }
+
+    proto = &vm->prototypes[NJS_OBJ_TYPE_TEXT_ENCODER].object;
+
+    njs_lvlhsh_init(&ov->object.hash);
+    njs_lvlhsh_init(&ov->object.shared_hash);
+    ov->object.type = NJS_OBJECT_VALUE;
+    ov->object.shared = 0;
+    ov->object.extensible = 1;
+    ov->object.error_data = 0;
+    ov->object.fast_array = 0;
+    ov->object.__proto__ = proto;
+    ov->object.slots = NULL;
+
+    njs_set_data(&ov->value, NULL, NJS_DATA_TAG_TEXT_ENCODER);
+    njs_set_object_value(&vm->retval, ov);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_text_encoder_encode(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
+{
+    u_char                *dst;
+    int64_t               size;
+    uint32_t              cp;
+    njs_int_t             ret;
+    njs_value_t           *this, *input, value;
+    const u_char          *p, *start, *end;
+    njs_string_prop_t     prop;
+    njs_typed_array_t     *array;
+    njs_unicode_decode_t  ctx;
+
+    this = njs_argument(args, 0);
+
+    if (njs_slow_path(!njs_is_object_data(this, NJS_DATA_TAG_TEXT_ENCODER))) {
+        njs_type_error(vm, "\"this\" is not a TextEncoder");
+        return NJS_ERROR;
+    }
+
+    start = NULL;
+    end = NULL;
+
+    if (nargs > 1) {
+        input = njs_argument(args, 1);
+
+        if (njs_slow_path(!njs_is_string(input))) {
+            ret = njs_value_to_string(vm, input, input);
+            if (njs_slow_path(ret != NJS_OK)) {
+                return ret;
+            }
+        }
+
+        (void) njs_string_prop(&prop, input);
+
+        if (prop.length != 0) {
+            return njs_text_encoder_encode_utf8(vm, &prop);
+        }
+
+        start = prop.start;
+        end = start + prop.size;
+    }
+
+    p = start;
+
+    cp = 0;
+    size = 0;
+
+    njs_utf8_decode_init(&ctx);
+
+    while (p < end) {
+        cp = njs_utf8_decode(&ctx, &p, end);
+
+        if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+            if (cp == NJS_UNICODE_CONTINUE) {
+                continue;
+            }
+
+            cp = NJS_UNICODE_REPLACEMENT;
+        }
+
+        size += njs_utf8_size(cp);
+    }
+
+    if (cp == NJS_UNICODE_CONTINUE) {
+        size += njs_utf8_size(NJS_UNICODE_REPLACEMENT);
+    }
+
+    njs_set_number(&value, size);
+
+    array = njs_typed_array_alloc(vm, &value, 1, NJS_OBJ_TYPE_UINT8_ARRAY);
+    if (njs_slow_path(array == NULL)) {
+        return NJS_ERROR;
+    }
+
+    dst = njs_typed_array_buffer(array)->u.u8;
+    njs_utf8_decode_init(&ctx);
+
+    while (start < end) {
+        cp = njs_utf8_decode(&ctx, &start, end);
+
+        if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+            if (cp == NJS_UNICODE_CONTINUE) {
+                continue;
+            }
+
+            cp = NJS_UNICODE_REPLACEMENT;
+        }
+
+        dst = njs_utf8_encode(dst, cp);
+    }
+
+    if (cp == NJS_UNICODE_CONTINUE) {
+        (void) njs_utf8_encode(dst, NJS_UNICODE_REPLACEMENT);
+    }
+
+    njs_set_typed_array(&vm->retval, array);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_text_encoder_encode_utf8(njs_vm_t *vm, njs_string_prop_t *prop)
+{
+    njs_value_t        value;
+    njs_typed_array_t  *array;
+
+    njs_set_number(&value, prop->size);
+
+    array = njs_typed_array_alloc(vm, &value, 1, NJS_OBJ_TYPE_UINT8_ARRAY);
+    if (njs_slow_path(array == NULL)) {
+        return NJS_ERROR;
+    }
+
+    memcpy(njs_typed_array_buffer(array)->u.u8, prop->start, prop->size);
+
+    njs_set_typed_array(&vm->retval, array);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_text_encoder_encode_into(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
+{
+    u_char                *to, *to_end;
+    size_t                size;
+    uint32_t              cp;
+    njs_int_t             ret;
+    njs_str_t             str;
+    njs_value_t           *this, *input, *dest, retval, read, written;
+    const u_char          *start, *end;
+    njs_typed_array_t     *array;
+    njs_unicode_decode_t  ctx;
+
+    static const njs_value_t  read_str = njs_string("read");
+    static const njs_value_t  written_str = njs_string("written");
+
+    this = njs_argument(args, 0);
+    input = njs_arg(args, nargs, 1);
+    dest = njs_arg(args, nargs, 2);
+
+    if (njs_slow_path(!njs_is_object_data(this, NJS_DATA_TAG_TEXT_ENCODER))) {
+        njs_type_error(vm, "\"this\" is not a TextEncoder");
+        return NJS_ERROR;
+    }
+
+    if (njs_slow_path(!njs_is_string(input))) {
+        ret = njs_value_to_string(vm, &retval, input);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+
+        input = &retval;
+    }
+
+    if (njs_slow_path(!njs_is_typed_array_uint8(dest))) {
+        njs_type_error(vm, "The \"destination\" argument must be an instance "
+                       "of Uint8Array");
+        return NJS_ERROR;
+    }
+
+    njs_string_get(input, &str);
+
+    start = str.start;
+    end = start + str.length;
+
+    array = njs_typed_array(dest);
+    to = njs_typed_array_buffer(array)->u.u8;
+    to_end = to + array->byte_length;
+
+    cp = 0;
+    njs_set_number(&read, 0);
+    njs_set_number(&written, 0);
+
+    njs_utf8_decode_init(&ctx);
+
+    while (start < end) {
+        cp = njs_utf8_decode(&ctx, &start, end);
+
+        if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+            cp = NJS_UNICODE_REPLACEMENT;
+        }
+
+        size = njs_utf8_size(cp);
+
+        if (to + size > to_end) {
+            break;
+        }
+
+        njs_number(&read) += (cp > 0xFFFF) ? 2 : 1;
+        njs_number(&written) += size;
+
+        to = njs_utf8_encode(to, cp);
+    }
+
+    return njs_vm_object_alloc(vm, &vm->retval, &read_str, &read,
+                               &written_str, &written, NULL);
+}
+
+
+static const njs_object_prop_t  njs_text_encoder_properties[] =
+{
+    {
+        .type = NJS_PROPERTY_HANDLER,
+        .name = njs_string("constructor"),
+        .value = njs_prop_handler(njs_object_prototype_create_constructor),
+        .writable = 1,
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("encoding"),
+        .value = njs_string("utf-8"),
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("encode"),
+        .value = njs_native_function(njs_text_encoder_encode, 0),
+        .writable = 1,
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("encodeInto"),
+        .value = njs_native_function(njs_text_encoder_encode_into, 2),
+        .writable = 1,
+        .configurable = 1,
+    },
+};
+
+
+const njs_object_init_t  njs_text_encoder_init = {
+    njs_text_encoder_properties,
+    njs_nitems(njs_text_encoder_properties),
+};
+
+
+static const njs_object_prop_t  njs_text_encoder_constructor_properties[] =
+{
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("name"),
+        .value = njs_string("TextEncoder"),
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("length"),
+        .value = njs_value(NJS_NUMBER, 0, 0.0),
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY_HANDLER,
+        .name = njs_string("prototype"),
+        .value = njs_prop_handler(njs_object_prototype_create),
+    },
+};
+
+
+const njs_object_init_t  njs_text_encoder_constructor_init = {
+    njs_text_encoder_constructor_properties,
+    njs_nitems(njs_text_encoder_constructor_properties),
+};
+
+
+const njs_object_type_init_t  njs_text_encoder_type_init = {
+   .constructor = njs_native_ctor(njs_text_encoder_constructor, 0, 0),
+   .prototype_props = &njs_text_encoder_init,
+   .constructor_props = &njs_text_encoder_constructor_init,
+   .prototype_value = { .object = { .type = NJS_OBJECT } },
+};
+
+
+static njs_int_t
+njs_text_decoder_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
+{
+    njs_int_t              ret;
+    njs_object_t           *proto;
+    njs_object_value_t     *ov;
+    njs_encoding_decode_t  *data;
+
+    if (!vm->top_frame->ctor) {
+        njs_type_error(vm, "Constructor of TextDecoder requires 'new'");
+        return NJS_ERROR;
+    }
+
+    ov = njs_mp_alloc(vm->mem_pool, sizeof(njs_object_value_t)
+                                    + sizeof(njs_encoding_decode_t));
+    if (njs_slow_path(ov == NULL)) {
+        njs_memory_error(vm);
+        return NJS_ERROR;
+    }
+
+    proto = &vm->prototypes[NJS_OBJ_TYPE_TEXT_DECODER].object;
+
+    njs_lvlhsh_init(&ov->object.hash);
+    njs_lvlhsh_init(&ov->object.shared_hash);
+    ov->object.type = NJS_OBJECT_VALUE;
+    ov->object.shared = 0;
+    ov->object.extensible = 1;
+    ov->object.error_data = 0;
+    ov->object.fast_array = 0;
+    ov->object.__proto__ = proto;
+    ov->object.slots = NULL;
+
+    data = (njs_encoding_decode_t *) ((uint8_t *) ov
+                                      + sizeof(njs_object_value_t));
+
+    ret = njs_text_decoder_arg_encoding(vm, args, nargs, data);
+    if (njs_slow_path(ret != NJS_OK)) {
+        return ret;
+    }
+
+    ret = njs_text_decoder_arg_options(vm, args, nargs, data);
+    if (njs_slow_path(ret != NJS_OK)) {
+        return ret;
+    }
+
+    data->codepoint = 0;
+    njs_utf8_decode_init(&data->ctx);
+
+    njs_set_data(&ov->value, data, NJS_DATA_TAG_TEXT_DECODER);
+    njs_set_object_value(&vm->retval, ov);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_text_decoder_arg_encoding(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_encoding_decode_t *data)
+{
+    njs_str_t             str;
+    njs_int_t             ret;
+    njs_value_t           *value;
+    njs_encoding_label_t  *label;
+
+    if (nargs < 2) {
+        data->encoding = NJS_ENCODING_UTF8;
+        return NJS_OK;
+    }
+
+    value = njs_argument(args, 1);
+
+    if (njs_slow_path(!njs_is_string(value))) {
+        ret = njs_value_to_string(vm, value, value);
+        if (njs_slow_path(ret != NJS_OK)) {
+            return ret;
+        }
+    }
+
+    njs_string_get(value, &str);
+
+    for (label = &njs_encoding_labels[0]; label->name.length != 0; label++) {
+        if (njs_strstr_eq(&str, &label->name)) {
+            data->encoding = label->encoding;
+            return NJS_OK;
+        }
+    }
+
+    njs_range_error(vm, "The \"%V\" encoding is not supported", &str);
+
+    return NJS_ERROR;
+}
+
+
+static njs_int_t
+njs_text_decoder_arg_options(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_encoding_decode_t *data)
+{
+    njs_int_t    ret;
+    njs_value_t  retval, *value;
+
+    static const njs_value_t  fatal_str = njs_string("fatal");
+    static const njs_value_t  ignore_bom_str = njs_string("ignoreBOM");
+
+    if (nargs < 3) {
+        data->fatal = 0;
+        data->ignore_bom = 0;
+
+        return NJS_OK;
+    }
+
+    value = njs_argument(args, 2);
+
+    if (njs_slow_path(!njs_is_object(value))) {
+        njs_type_error(vm, "The \"options\" argument must be of type object");
+        return NJS_ERROR;
+    }
+
+    ret = njs_value_property(vm, value, njs_value_arg(&fatal_str), &retval);
+    if (njs_slow_path(ret == NJS_ERROR)) {
+        return ret;
+    }
+
+    data->fatal = njs_bool(&retval);
+
+    ret = njs_value_property(vm, value, njs_value_arg(&ignore_bom_str),
+                             &retval);
+    if (njs_slow_path(ret == NJS_ERROR)) {
+        return ret;
+    }
+
+    data->ignore_bom = njs_bool(&retval);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_text_decoder_encoding(njs_vm_t *vm, njs_object_prop_t *prop,
+    njs_value_t *value, njs_value_t *setval, njs_value_t *retval)
+{
+    njs_encoding_decode_t  *data;
+
+    static const njs_value_t  utf8_str = njs_string("utf-8");
+
+    if (njs_slow_path(!njs_is_object_data(value, NJS_DATA_TAG_TEXT_DECODER))) {
+        njs_set_undefined(retval);
+        return NJS_DECLINED;
+    }
+
+    data = njs_object_data(value);
+
+    switch (data->encoding) {
+    case NJS_ENCODING_UTF8:
+        *retval = utf8_str;
+        break;
+
+    default:
+        njs_type_error(vm, "unknown encoding");
+        return NJS_ERROR;
+    }
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_text_decoder_fatal(njs_vm_t *vm, njs_object_prop_t *prop,
+    njs_value_t *value, njs_value_t *setval, njs_value_t *retval)
+{
+    njs_encoding_decode_t  *data;
+
+    if (njs_slow_path(!njs_is_object_data(value, NJS_DATA_TAG_TEXT_DECODER))) {
+        njs_set_undefined(retval);
+        return NJS_DECLINED;
+    }
+
+    data = njs_object_data(value);
+
+    njs_set_boolean(retval, data->fatal);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_text_decoder_ignore_bom(njs_vm_t *vm, njs_object_prop_t *prop,
+    njs_value_t *value, njs_value_t *setval, njs_value_t *retval)
+{
+    njs_encoding_decode_t  *data;
+
+    if (njs_slow_path(!njs_is_object_data(value, NJS_DATA_TAG_TEXT_DECODER))) {
+        njs_set_undefined(retval);
+        return NJS_DECLINED;
+    }
+
+    data = njs_object_data(value);
+
+    njs_set_boolean(retval, data->ignore_bom);
+
+    return NJS_OK;
+}
+
+
+static njs_int_t
+njs_text_decoder_decode(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_index_t unused)
+{
+    u_char                   *dst;
+    uint32_t                 length, cp;
+    uint64_t                 size;
+    njs_int_t                ret;
+    njs_bool_t               stream;
+    njs_value_t              retval, *this, *typed_array, *options;
+    const u_char             *start, *end, *p;
+    njs_unicode_decode_t     ctx;
+    njs_encoding_decode_t    *data;
+    const njs_typed_array_t  *array;
+
+    static const njs_value_t  stream_str = njs_string("stream");
+
+    start = NULL;
+    end = NULL;
+
+    stream = 0;
+
+    this = njs_argument(args, 0);
+
+    if (njs_slow_path(!njs_is_object_data(this, NJS_DATA_TAG_TEXT_DECODER))) {
+        njs_type_error(vm, "\"this\" is not a TextDecoder");
+        return NJS_ERROR;
+    }
+
+    if (njs_fast_path(nargs > 1)) {
+        typed_array = njs_argument(args, 1);
+        if (njs_slow_path(!njs_is_typed_array(typed_array))) {
+            njs_type_error(vm, "The \"input\" argument must be an instance "
+                           "of TypedArray");
+            return NJS_ERROR;
+        }
+
+        array = njs_typed_array(typed_array);
+
+        start = array->buffer->u.u8;
+        end = start + array->byte_length;
+    }
+
+    if (nargs > 2) {
+        options = njs_argument(args, 2);
+
+        if (njs_slow_path(!njs_is_object(options))) {
+            njs_type_error(vm, "The \"options\" argument must be "
+                           "of type object");
+            return NJS_ERROR;
+        }
+
+        ret = njs_value_property(vm, options, njs_value_arg(&stream_str),
+                                 &retval);
+        if (njs_slow_path(ret == NJS_ERROR)) {
+            return ret;
+        }
+
+        stream = njs_bool(&retval);
+    }
+
+    data = njs_object_data(this);
+
+    ctx = data->ctx;
+    cp = data->codepoint;
+
+    size = 0;
+    length = 0;
+
+    p = start;
+
+    /* Looking for BOM. */
+
+    if (!data->ignore_bom && p + 3 <= end) {
+        cp = njs_utf8_decode(&ctx, &p, end);
+
+        if (cp == NJS_UNICODE_BOM) {
+            start = p;
+
+        } else {
+            p = start;
+        }
+    }
+
+    while (p < end) {
+        cp = njs_utf8_decode(&ctx, &p, end);
+
+        if (njs_slow_path(cp > NJS_UNICODE_MAX_CODEPOINT)) {
+            if (cp == NJS_UNICODE_CONTINUE) {
+                break;
+            }
+
+            if (data->fatal) {
+                goto fatal;
+            }
+
+            cp = NJS_UNICODE_REPLACEMENT;
+        }
+
+        size += njs_utf8_size(cp);
+        length++;
+    }
+
+    if (cp == NJS_UNICODE_CONTINUE && !stream) {
+        if (data->fatal) {
+            goto fatal;
+        }
+
+        size += njs_utf8_size(NJS_UNICODE_REPLACEMENT);
+        length++;
+    }
+
+    dst = njs_string_alloc(vm, &vm->retval, size, length);
+    if (njs_slow_path(dst == NULL)) {
+        return NJS_ERROR;
+    }
+
+    while (start < end) {
+        cp = njs_utf8_decode(&data->ctx, &start, end);
+
+        if (cp > NJS_UNICODE_MAX_CODEPOINT) {
+            if (cp == NJS_UNICODE_CONTINUE) {
+                break;
+            }
+
+            cp = NJS_UNICODE_REPLACEMENT;
+        }
+
+        dst = njs_utf8_encode(dst, cp);
+    }
+
+    if (stream) {
+        data->codepoint = cp;
+        return NJS_OK;
+    }
+
+    if (cp == NJS_UNICODE_CONTINUE) {
+        (void) njs_utf8_encode(dst, NJS_UNICODE_REPLACEMENT);
+    }
+
+    data->codepoint = 0;
+
+    njs_utf8_decode_init(&data->ctx);
+
+    return NJS_OK;
+
+fatal:
+
+    njs_type_error(vm, "The encoded data was not valid");
+
+    return NJS_ERROR;
+}
+
+
+static const njs_object_prop_t  njs_text_decoder_properties[] =
+{
+    {
+        .type = NJS_PROPERTY_HANDLER,
+        .name = njs_string("constructor"),
+        .value = njs_prop_handler(njs_object_prototype_create_constructor),
+        .writable = 1,
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY_HANDLER,
+        .name = njs_string("encoding"),
+        .value = njs_prop_handler(njs_text_decoder_encoding),
+    },
+
+    {
+        .type = NJS_PROPERTY_HANDLER,
+        .name = njs_string("fatal"),
+        .value = njs_prop_handler(njs_text_decoder_fatal),
+    },
+
+    {
+        .type = NJS_PROPERTY_HANDLER,
+        .name = njs_string("ignoreBOM"),
+        .value = njs_prop_handler(njs_text_decoder_ignore_bom),
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("decode"),
+        .value = njs_native_function(njs_text_decoder_decode, 0),
+        .writable = 1,
+        .configurable = 1,
+    },
+};
+
+
+const njs_object_init_t  njs_text_decoder_init = {
+    njs_text_decoder_properties,
+    njs_nitems(njs_text_decoder_properties),
+};
+
+
+static const njs_object_prop_t  njs_text_decoder_constructor_properties[] =
+{
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("name"),
+        .value = njs_string("TextDecoder"),
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY,
+        .name = njs_string("length"),
+        .value = njs_value(NJS_NUMBER, 0, 0.0),
+        .configurable = 1,
+    },
+
+    {
+        .type = NJS_PROPERTY_HANDLER,
+        .name = njs_string("prototype"),
+        .value = njs_prop_handler(njs_object_prototype_create),
+    },
+};
+
+
+const njs_object_init_t  njs_text_decoder_constructor_init = {
+    njs_text_decoder_constructor_properties,
+    njs_nitems(njs_text_decoder_constructor_properties),
+};
+
+
+const njs_object_type_init_t  njs_text_decoder_type_init = {
+   .constructor = njs_native_ctor(njs_text_decoder_constructor, 0, 0),
+   .prototype_props = &njs_text_decoder_init,
+   .constructor_props = &njs_text_decoder_constructor_init,
+   .prototype_value = { .object = { .type = NJS_OBJECT } },
+};
diff -r 0fad09ddb37a -r bcd1a41c6a67 src/njs_encoding.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/njs_encoding.h	Tue Jul 28 16:58:59 2020 +0300
@@ -0,0 +1,14 @@
+
+/*
+ * Copyright (C) Alexander Borisov
+ * Copyright (C) NGINX, Inc.
+ */
+
+#ifndef _NJS_ENCODING_H_INCLUDED_
+#define _NJS_ENCODING_H_INCLUDED_
+
+extern const njs_object_type_init_t  njs_text_encoder_type_init;
+extern const njs_object_type_init_t  njs_text_decoder_type_init;
+
+
+#endif /* _NJS_ENCODING_H_INCLUDED_ */
diff -r 0fad09ddb37a -r bcd1a41c6a67 src/njs_main.h
--- a/src/njs_main.h	Thu Jul 30 17:47:05 2020 +0000
+++ b/src/njs_main.h	Tue Jul 28 16:58:59 2020 +0300
@@ -73,6 +73,7 @@
 
 #include <njs_math.h>
 #include <njs_json.h>
+#include <njs_encoding.h>
 
 #include <njs_timer.h>
 #include <njs_module.h>
diff -r 0fad09ddb37a -r bcd1a41c6a67 src/njs_object_hash.h
--- a/src/njs_object_hash.h	Thu Jul 30 17:47:05 2020 +0000
+++ b/src/njs_object_hash.h	Tue Jul 28 16:58:59 2020 +0300
@@ -750,4 +750,34 @@
         'd'), 'A'), 'r'), 'r'), 'a'), 'y')
 
 
+#define NJS_TEXT_DECODER_HASH                                                 \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(NJS_DJB_HASH_INIT,                                       \
+        'T'), 'e'), 'x'), 't'), 'D'), 'e'), 'c'), 'o'), 'd'), 'e'), 'r')
+
+
+#define NJS_TEXT_ENCODER_HASH                                                 \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(                                                         \
+    njs_djb_hash_add(NJS_DJB_HASH_INIT,                                       \
+        'T'), 'e'), 'x'), 't'), 'E'), 'n'), 'c'), 'o'), 'd'), 'e'), 'r')
+
+
 #endif /* _NJS_OBJECT_HASH_H_INCLUDED_ */
diff -r 0fad09ddb37a -r bcd1a41c6a67 src/njs_typed_array.c
--- a/src/njs_typed_array.c	Thu Jul 30 17:47:05 2020 +0000
+++ b/src/njs_typed_array.c	Tue Jul 28 16:58:59 2020 +0300
@@ -8,9 +8,9 @@
 #include <njs_main.h>
 
 
-static njs_int_t
-njs_typed_array_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
-    njs_index_t magic)
+njs_typed_array_t *
+njs_typed_array_alloc(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
+    njs_object_type_t type)
 {
     double              num;
     int64_t             i, length;
@@ -19,7 +19,6 @@ njs_typed_array_constructor(njs_vm_t *vm
     njs_int_t           ret;
     njs_value_t         *value, prop;
     njs_array_t         *src_array;
-    njs_object_type_t   type;
     njs_typed_array_t   *array, *src_tarray;
     njs_array_buffer_t  *buffer;
 
@@ -31,54 +30,48 @@ njs_typed_array_constructor(njs_vm_t *vm
     src_array = NULL;
     src_tarray = NULL;
 
-    type = magic;
     element_size = njs_typed_array_element_size(type);
 
-    if (!vm->top_frame->ctor) {
-        njs_type_error(vm, "Constructor of TypedArray requires 'new'");
-        return NJS_ERROR;
-    }
-
-    value = njs_arg(args, nargs, 1);
+    value = njs_arg(args, nargs, 0);
 
     if (njs_is_array_buffer(value)) {
         buffer = njs_array_buffer(value);
 
-        ret = njs_value_to_index(vm, njs_arg(args, nargs, 2), &offset);
+        ret = njs_value_to_index(vm, njs_arg(args, nargs, 1), &offset);
         if (njs_slow_path(ret != NJS_OK)) {
-            return NJS_ERROR;
+            return NULL;
         }
 
         if (njs_slow_path((offset % element_size) != 0)) {
             njs_range_error(vm, "start offset must be multiple of %uD",
                             element_size);
-            return NJS_ERROR;
+            return NULL;
         }
 
-        if (!njs_is_undefined(njs_arg(args, nargs, 3))) {
-            ret = njs_value_to_index(vm, njs_argument(args, 3), &size);
+        if (!njs_is_undefined(njs_arg(args, nargs, 2))) {
+            ret = njs_value_to_index(vm, njs_argument(args, 2), &size);
             if (njs_slow_path(ret != NJS_OK)) {
-                return NJS_ERROR;
+                return NULL;
             }
 
             size *= element_size;
 
             if (njs_slow_path((offset + size) > buffer->size)) {
                 njs_range_error(vm, "Invalid typed array length: %uL", size);
-                return NJS_ERROR;
+                return NULL;
             }
 
         } else {
             if (njs_slow_path((buffer->size % element_size) != 0)) {