From pluknet at nginx.com Tue Jun 1 11:31:57 2021 From: pluknet at nginx.com (Sergey Kandaurov) Date: Tue, 1 Jun 2021 14:31:57 +0300 Subject: [nginx-quic] fix qpack header null value issue In-Reply-To: References: Message-ID: <0171101B-0F15-4429-B774-400BF274DB73@nginx.com> > On 28 May 2021, at 07:02, sun edward wrote: > > description: > when header with a null value,need to reset st->value,otherwise it is taking previous header field's value > Thanks, a slightly different version committed: https://hg.nginx.org/nginx-quic/rev/3509b9dcfb47 [..] -- Sergey Kandaurov From mdounin at mdounin.ru Tue Jun 1 14:42:52 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 01 Jun 2021 14:42:52 +0000 Subject: [nginx] SSL: ngx_ssl_shutdown() rework. Message-ID: details: https://hg.nginx.org/nginx/rev/fecf645ff2f8 branches: changeset: 7870:fecf645ff2f8 user: Maxim Dounin date: Tue Jun 01 17:37:49 2021 +0300 description: SSL: ngx_ssl_shutdown() rework. Instead of calling SSL_free() with each return point, introduced a single place where cleanup happens. As a positive side effect, this fixes two potential memory leaks on ngx_handle_read_event() and ngx_handle_write_event() errors where there were no SSL_free() calls (though unlikely practical, as errors there are only expected to happen due to bugs or kernel issues). diffstat: src/event/ngx_event_openssl.c | 45 +++++++++++++++++++++--------------------- 1 files changed, 22 insertions(+), 23 deletions(-) diffs (95 lines): diff -r d61d590ac826 -r fecf645ff2f8 src/event/ngx_event_openssl.c --- a/src/event/ngx_event_openssl.c Sun May 30 12:26:00 2021 +0300 +++ b/src/event/ngx_event_openssl.c Tue Jun 01 17:37:49 2021 +0300 @@ -2896,9 +2896,12 @@ ngx_int_t ngx_ssl_shutdown(ngx_connection_t *c) { int n, sslerr, mode; + ngx_int_t rc; ngx_err_t err; ngx_uint_t tries; + rc = NGX_OK; + ngx_ssl_ocsp_cleanup(c); if (SSL_in_init(c->ssl->connection)) { @@ -2908,11 +2911,7 @@ ngx_ssl_shutdown(ngx_connection_t *c) * Avoid calling SSL_shutdown() if handshake wasn't completed. */ - SSL_free(c->ssl->connection); - c->ssl = NULL; - c->recv = ngx_recv; - - return NGX_OK; + goto done; } if (c->timedout || c->error || c->buffered) { @@ -2954,11 +2953,7 @@ ngx_ssl_shutdown(ngx_connection_t *c) ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_shutdown: %d", n); if (n == 1) { - SSL_free(c->ssl->connection); - c->ssl = NULL; - c->recv = ngx_recv; - - return NGX_OK; + goto done; } if (n == 0 && tries-- > 1) { @@ -2984,11 +2979,11 @@ ngx_ssl_shutdown(ngx_connection_t *c) } if (ngx_handle_read_event(c->read, 0) != NGX_OK) { - return NGX_ERROR; + goto failed; } if (ngx_handle_write_event(c->write, 0) != NGX_OK) { - return NGX_ERROR; + goto failed; } ngx_add_timer(c->read, 3000); @@ -2997,23 +2992,27 @@ ngx_ssl_shutdown(ngx_connection_t *c) } if (sslerr == SSL_ERROR_ZERO_RETURN || ERR_peek_error() == 0) { - SSL_free(c->ssl->connection); - c->ssl = NULL; - c->recv = ngx_recv; - - return NGX_OK; + goto done; } err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0; ngx_ssl_connection_error(c, sslerr, err, "SSL_shutdown() failed"); - SSL_free(c->ssl->connection); - c->ssl = NULL; - c->recv = ngx_recv; - - return NGX_ERROR; - } + break; + } + +failed: + + rc = NGX_ERROR; + +done: + + SSL_free(c->ssl->connection); + c->ssl = NULL; + c->recv = ngx_recv; + + return rc; } From mdounin at mdounin.ru Tue Jun 1 14:42:55 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 01 Jun 2021 14:42:55 +0000 Subject: [nginx] Fixed SSL logging with lingering close. Message-ID: details: https://hg.nginx.org/nginx/rev/5f765427c17a branches: changeset: 7871:5f765427c17a user: Maxim Dounin date: Tue Jun 01 17:37:51 2021 +0300 description: Fixed SSL logging with lingering close. Recent fixes to SSL shutdown with lingering close (554c6ae25ffc, 1.19.5) broke logging of SSL variables. To make sure logging of SSL variables works properly, avoid freeing c->ssl when doing an SSL shutdown before lingering close. Reported by Reinis Rozitis (http://mailman.nginx.org/pipermail/nginx/2021-May/060670.html). diffstat: src/event/ngx_event_openssl.c | 6 ++++++ src/event/ngx_event_openssl.h | 1 + src/http/ngx_http_request.c | 2 ++ 3 files changed, 9 insertions(+), 0 deletions(-) diffs (39 lines): diff -r fecf645ff2f8 -r 5f765427c17a src/event/ngx_event_openssl.c --- a/src/event/ngx_event_openssl.c Tue Jun 01 17:37:49 2021 +0300 +++ b/src/event/ngx_event_openssl.c Tue Jun 01 17:37:51 2021 +0300 @@ -3008,6 +3008,12 @@ failed: done: + if (c->ssl->shutdown_without_free) { + c->ssl->shutdown_without_free = 0; + c->recv = ngx_recv; + return rc; + } + SSL_free(c->ssl->connection); c->ssl = NULL; c->recv = ngx_recv; diff -r fecf645ff2f8 -r 5f765427c17a src/event/ngx_event_openssl.h --- a/src/event/ngx_event_openssl.h Tue Jun 01 17:37:49 2021 +0300 +++ b/src/event/ngx_event_openssl.h Tue Jun 01 17:37:51 2021 +0300 @@ -100,6 +100,7 @@ struct ngx_ssl_connection_s { unsigned buffer:1; unsigned no_wait_shutdown:1; unsigned no_send_shutdown:1; + unsigned shutdown_without_free:1; unsigned handshake_buffer_set:1; unsigned try_early_data:1; unsigned in_early:1; diff -r fecf645ff2f8 -r 5f765427c17a src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c Tue Jun 01 17:37:49 2021 +0300 +++ b/src/http/ngx_http_request.c Tue Jun 01 17:37:51 2021 +0300 @@ -3400,6 +3400,8 @@ ngx_http_set_lingering_close(ngx_connect if (c->ssl) { ngx_int_t rc; + c->ssl->shutdown_without_free = 1; + rc = ngx_ssl_shutdown(c); if (rc == NGX_ERROR) { From Steffen.Kiess at ipvs.uni-stuttgart.de Tue Jun 1 16:39:42 2021 From: Steffen.Kiess at ipvs.uni-stuttgart.de (=?UTF-8?Q?Steffen_Kie=c3=9f?=) Date: Tue, 1 Jun 2021 18:39:42 +0200 Subject: [PATCH] SSL: export channel binding values as variables In-Reply-To: References: <6c52eeee-107d-1582-9ada-9e10cb9518c0@ipvs.uni-stuttgart.de> Message-ID: <513efb5c-6ed6-b563-bd98-bf70ff4e7c1d@ipvs.uni-stuttgart.de> Hello, On 01.06.21 01:08, Maxim Dounin wrote: > Hello! > > On Mon, May 31, 2021 at 09:41:42PM +0200, Steffen Kie? wrote: > >> On 31.05.21 18:36, Maxim Dounin wrote: >>> >>> Thanks for the patch. You may want to elaborate a bit more on how >>> do you expect these variables to be used. >>> >>> [...] >>> >> >> These variables can be used to implement authentication with channel >> binding in an http application. > > [...] > >> I've attached a flask application + a client which shows how this can be >> used, the required configuration in NGINX (when using fastcgi) is: > > So, you expect these variables to be used by application > developers to implement some (currently not implemented) > authentication with channel binding in HTTP, and that's the only > use case you consider, correct? One use case is channel binding for RFC4559. This already exists (Microsoft IIS implements the server side with channel binding, Microsoft Edge implements the client side with channel binding). There are also other implementations without channel binding (firefox and libcurl as clients for example) but this is in general vulnerable to MitM attacks, even when using HTTPS (because an attacker who can cause a client to do authentication over HTTP can then reuse the authentication data for doing authentication over HTTPS). RFC4559 has some rather weird design decisions (like doing authentication per-connections instead of per-request) but it is used in real life. On the other RFC4559 uses the tls-server-end-point channel binding which can also be done without any support by NGINX (by providing the server certificate to the web application). > > Note that HTTP provides no guarantees about channels, that is, > connections, and trying to use channel binding is expected to > break operation over HTTP, especially in complex setups when using > proxies or reverse proxies, such as nginx. Further, invalid > assumptions about guarantees in HTTP can easily cause security > issues, by incorrectly authenticating unrelated requests on the > connection. Basically the same set of issues as already seen with > Microsoft's mis-designed NTLM authentication which doesn't work > through proxies. Channel binding is for HTTPS only, so there should not be any issues with proxies. Reverse proxies and load balancers obviously will make this more complicated (because the channel binding information would have to be forwarded to the web application in way which an attacker cannot spoof), but in simple cases (e.g. using NGINX as a frontend which forwards requests to an application over FastCGI or the uwsgi protocol) this should not cause any problems. Note that using channel bindings does not require you to do per-connection authentication; you still can do per-request authentication and only gain an assurance that no attack is performing a MitM attack on the TLS connection. > > Given that, it might not be a good idea to provide such variables. > Best regards, Steffen Kie? From xeioex at nginx.com Wed Jun 2 14:10:19 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Wed, 02 Jun 2021 14:10:19 +0000 Subject: [njs] Fixed integer-overflow in MakeDay(). Message-ID: details: https://hg.nginx.org/njs/rev/befc2827d2d2 branches: changeset: 1648:befc2827d2d2 user: Dmitry Volyntsev date: Wed Jun 02 13:25:32 2021 +0000 description: Fixed integer-overflow in MakeDay(). Found by OSS-Fuzz. diffstat: src/njs_date.c | 7 ++++++- src/test/njs_unit_test.c | 3 +++ 2 files changed, 9 insertions(+), 1 deletions(-) diffs (33 lines): diff -r de189c66c757 -r befc2827d2d2 src/njs_date.c --- a/src/njs_date.c Mon May 31 06:55:34 2021 +0000 +++ b/src/njs_date.c Wed Jun 02 13:25:32 2021 +0000 @@ -124,10 +124,15 @@ njs_make_day(int64_t yr, int64_t month, double days; int64_t i, ym, mn, md; + static const int min_year = -271821; + static const int max_year = 275760; static const int month_days[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - if (yr < -271822 || yr > 275761) { + if (yr < min_year || yr > max_year + || month < (min_year * 12) || month > (max_year * 12) + || date < (min_year * 12 * 366) || date > (max_year * 12 * 366)) + { return NAN; } diff -r de189c66c757 -r befc2827d2d2 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Mon May 31 06:55:34 2021 +0000 +++ b/src/test/njs_unit_test.c Wed Jun 02 13:25:32 2021 +0000 @@ -15211,6 +15211,9 @@ static njs_unit_test_t njs_test[] = { njs_str("new Date(NaN)"), njs_str("Invalid Date") }, + { njs_str("new Date(0, 9e99)"), + njs_str("Invalid Date") }, + #ifndef NJS_SUNC { njs_str("new Date(-0).getTime()"), njs_str("0") }, From xeioex at nginx.com Wed Jun 2 14:10:21 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Wed, 02 Jun 2021 14:10:21 +0000 Subject: [njs] Improved arguments parsing in Date constructor. Message-ID: details: https://hg.nginx.org/njs/rev/687d9eacbe33 branches: changeset: 1649:687d9eacbe33 user: Dmitry Volyntsev date: Wed Jun 02 14:10:03 2021 +0000 description: Improved arguments parsing in Date constructor. diffstat: src/njs_date.c | 123 +++++++++++++++++++--------------------------- src/test/njs_unit_test.c | 9 +++ 2 files changed, 60 insertions(+), 72 deletions(-) diffs (197 lines): diff -r befc2827d2d2 -r 687d9eacbe33 src/njs_date.c --- a/src/njs_date.c Wed Jun 02 13:25:32 2021 +0000 +++ b/src/njs_date.c Wed Jun 02 14:10:03 2021 +0000 @@ -311,12 +311,47 @@ njs_destruct_date(double time, int64_t t static njs_int_t +njs_date_args(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, + int64_t tm[]) +{ + double num; + njs_int_t ret; + njs_uint_t i, n; + + njs_memzero(tm, NJS_DATE_MAX_FIELDS * sizeof(int64_t)); + + tm[NJS_DATE_DAY] = 1; + + n = njs_min(8, nargs); + + for (i = 1; i < n; i++) { + ret = njs_value_to_number(vm, &args[i], &num); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + if (!isfinite(num)) { + tm[NJS_DATE_YR] = INT64_MIN; + continue; + } + + tm[i] = njs_number_to_integer(num); + } + + if (tm[NJS_DATE_YR] >= 0 && tm[NJS_DATE_YR] < 100) { + tm[NJS_DATE_YR] += 1900; + } + + return NJS_OK; +} + + +static njs_int_t njs_date_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused) { - double num, time; + double time; njs_int_t ret; - njs_uint_t i, n; njs_date_t *date; int64_t tm[NJS_DATE_MAX_FIELDS]; @@ -350,40 +385,14 @@ njs_date_constructor(njs_vm_t *vm, njs_v } else { - time = NAN; - - njs_memzero(tm, NJS_DATE_MAX_FIELDS * sizeof(int64_t)); - - tm[NJS_DATE_DAY] = 1; - - n = njs_min(8, nargs); - - for (i = 1; i < n; i++) { - if (!njs_is_numeric(&args[i])) { - ret = njs_value_to_numeric(vm, &args[i], &args[i]); - if (njs_slow_path(ret != NJS_OK)) { - return ret; - } - } - - num = njs_number(&args[i]); - - if (isnan(num) || isinf(num)) { - goto done; - } - - tm[i] = num; - } - - if (tm[NJS_DATE_YR] >= 0 && tm[NJS_DATE_YR] < 100) { - tm[NJS_DATE_YR] += 1900; + ret = njs_date_args(vm, args, nargs, tm); + if (njs_slow_path(ret != NJS_OK)) { + return ret; } time = njs_make_date(tm, 1); } -done: - date = njs_mp_alloc(vm->mem_pool, sizeof(njs_date_t)); if (njs_slow_path(date == NULL)) { njs_memory_error(vm); @@ -412,47 +421,21 @@ static njs_int_t njs_date_utc(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused) { - double num, time; + double time; njs_int_t ret; - njs_uint_t i, n; int64_t tm[NJS_DATE_MAX_FIELDS]; time = NAN; if (nargs > 1) { - njs_memzero(tm, NJS_DATE_MAX_FIELDS * sizeof(int64_t)); - - tm[NJS_DATE_DAY] = 1; - - n = njs_min(8, nargs); - - for (i = 1; i < n; i++) { - if (!njs_is_numeric(&args[i])) { - ret = njs_value_to_numeric(vm, &args[i], &args[i]); - if (ret != NJS_OK) { - return ret; - } - } - - num = njs_number(&args[i]); - - if (isnan(num) || isinf(num)) { - goto done; - } - - tm[i] = num; - } - - /* Year. */ - if (tm[NJS_DATE_YR] >= 0 && tm[NJS_DATE_YR] < 100) { - tm[NJS_DATE_YR] += 1900; + ret = njs_date_args(vm, args, nargs, tm); + if (njs_slow_path(ret != NJS_OK)) { + return ret; } time = njs_make_date(tm, 0); } -done: - njs_set_number(&vm->retval, time); return NJS_OK; @@ -1385,21 +1368,17 @@ njs_date_prototype_set_fields(njs_vm_t * njs_destruct_date(time, tm, 0, magic & 0x40); do { - if (njs_slow_path(!njs_is_number(&args[i]))) { - ret = njs_value_to_numeric(vm, &args[i], &args[i]); - if (njs_slow_path(ret != NJS_OK)) { - return ret; - } + ret = njs_value_to_number(vm, &args[i++], &num); + if (njs_slow_path(ret != NJS_OK)) { + return ret; } - num = njs_number(&args[i++]); - - if (isnan(num) || isinf(num)) { - time = NAN; - goto done; + if (!isfinite(num)) { + tm[NJS_DATE_YR] = INT64_MIN; + continue; } - tm[since++] = num; + tm[since++] = njs_number_to_integer(num); } while (--left); diff -r befc2827d2d2 -r 687d9eacbe33 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Wed Jun 02 13:25:32 2021 +0000 +++ b/src/test/njs_unit_test.c Wed Jun 02 14:10:03 2021 +0000 @@ -14655,6 +14655,15 @@ static njs_unit_test_t njs_test[] = { njs_str("new Date(NaN)"), njs_str("Invalid Date") }, + { njs_str("new Date(1,undefined)"), + njs_str("Invalid Date") }, + + { njs_str("new Date(1,Infinity)"), + njs_str("Invalid Date") }, + + { njs_str("new Date(1,NaN)"), + njs_str("Invalid Date") }, + { njs_str("new Date(8.65e15)"), njs_str("Invalid Date") }, From mdounin at mdounin.ru Wed Jun 2 22:18:28 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 3 Jun 2021 01:18:28 +0300 Subject: [PATCH] SSL: export channel binding values as variables In-Reply-To: <513efb5c-6ed6-b563-bd98-bf70ff4e7c1d@ipvs.uni-stuttgart.de> References: <6c52eeee-107d-1582-9ada-9e10cb9518c0@ipvs.uni-stuttgart.de> <513efb5c-6ed6-b563-bd98-bf70ff4e7c1d@ipvs.uni-stuttgart.de> Message-ID: Hello! On Tue, Jun 01, 2021 at 06:39:42PM +0200, Steffen Kie? wrote: > On 01.06.21 01:08, Maxim Dounin wrote: > > Hello! > > > > On Mon, May 31, 2021 at 09:41:42PM +0200, Steffen Kie? wrote: > > > >> On 31.05.21 18:36, Maxim Dounin wrote: > >>> > >>> Thanks for the patch. You may want to elaborate a bit more on how > >>> do you expect these variables to be used. > >>> > >>> [...] > >>> > >> > >> These variables can be used to implement authentication with channel > >> binding in an http application. > > > > [...] > > > >> I've attached a flask application + a client which shows how this can be > >> used, the required configuration in NGINX (when using fastcgi) is: > > > > So, you expect these variables to be used by application > > developers to implement some (currently not implemented) > > authentication with channel binding in HTTP, and that's the only > > use case you consider, correct? > > One use case is channel binding for RFC4559. This already exists > (Microsoft IIS implements the server side with channel binding, > Microsoft Edge implements the client side with channel binding). There > are also other implementations without channel binding (firefox and > libcurl as clients for example) but this is in general vulnerable to > MitM attacks, even when using HTTPS (because an attacker who can cause a > client to do authentication over HTTP can then reuse the authentication > data for doing authentication over HTTPS). RFC4559 has some rather weird > design decisions (like doing authentication per-connections instead of > per-request) but it is used in real life. Thank you for the details. The "rather weird design decisions" is exactly what I was talking about: invalid assumptions about HTTP guarantees and the fact that it works in simple cases resulted in design decisions which made the RFC 4559 authentication mechanism incompatible with HTTP (see the RFC errata). > On the other RFC4559 uses the tls-server-end-point channel binding which > can also be done without any support by NGINX (by providing the server > certificate to the web application). The "without any suppport by nginx" looks like the way to go. -- Maxim Dounin http://mdounin.ru/ From tracey at archive.org Thu Jun 3 00:17:52 2021 From: tracey at archive.org (Tracey Jaquith) Date: Thu, 03 Jun 2021 00:17:52 +0000 Subject: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes Message-ID: <5da9c62fa61016600f2c.1622679472@vm-home1.archive.org> # HG changeset patch # User Tracey Jaquith # Date 1622678642 0 # Thu Jun 03 00:04:02 2021 +0000 # Node ID 5da9c62fa61016600f2c59982ae184e2811be427 # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 Add optional "&exact=1" CGI arg to show video between keyframes. archive.org has been using mod_h264_streaming with an "&exact=1" patch from me since 2013. We just moved to nginx mp4 module and are using this patch. The technique is to find the keyframe just before the desired "start" time, and send that down the wire so video playback can start immediately. Next calculate how many video samples are between the keyframe and desired "start" time and update the STTS atom where those samples move the duration from (typically) 1001 to 1. This way, initial unwanted video frames play at ~1/30,000s -- so visually the video & audio start playing immediately. You can see an example before/after here (nginx binary built with mp4 module + patch): https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30 https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30&exact=1 Tested on linux and macosx. I realize two var declarations should style-wise get moved "up" in the lower hooked in method. However, to minimize the changeset/patch, I figured we should at least start here and see what you folks think. (this is me: https://github.com/traceypooh ) diff -r 5f765427c17a -r 5da9c62fa610 src/http/modules/ngx_http_mp4_module.c --- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300 +++ b/src/http/modules/ngx_http_mp4_module.c Thu Jun 03 00:04:02 2021 +0000 @@ -2045,6 +2045,109 @@ u_char duration[4]; } ngx_mp4_stts_entry_t; +typedef struct { + uint32_t speedup_samples; + ngx_uint_t speedup_seconds; +} ngx_mp4_exact_t; + +static void +exact_video_adjustment(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_mp4_exact_t *exact) +{ + // will parse STTS -- time-to-sample atom + ngx_str_t value; + ngx_buf_t *stts_data; + ngx_buf_t *atom; + ngx_mp4_stts_entry_t *stts_entry, *stts_end; + uint32_t count, duration, j, n, sample_keyframe, sample_num; + uint64_t sample_time, seconds, start_seconds_closest_keyframe; + uint8_t is_keyframe; + + exact->speedup_samples = 0; + exact->speedup_seconds = 0; + + // if '&exact=1' CGI arg isn't present, do nothing + if (!(ngx_http_arg(mp4->request, (u_char *) "exact", 5, &value) == NGX_OK)) { + return; + } + + if (!trak->sync_samples_entries) { + // Highly unlikely video STSS got parsed and _every_ sample is a keyframe. + // However, if the case, we don't need to adjust the video at all. + return; + } + + // check HDLR atom to see if this trak is video or audio + atom = trak->out[NGX_HTTP_MP4_HDLR_ATOM].buf; + // 'vide' or 'soun' + if (!(atom->pos[16] == 'v' && + atom->pos[17] == 'i' && + atom->pos[18] == 'd' && + atom->pos[19] == 'e')) { + return; // do nothing if not video + } + + + stts_data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf; + stts_entry = (ngx_mp4_stts_entry_t *) stts_data->pos; + stts_end = (ngx_mp4_stts_entry_t *) stts_data->last; + + sample_num = 0; // they start at one + sample_time = 0; + start_seconds_closest_keyframe = 0; + while (stts_entry < stts_end) { + // STTS === time-to-sample atom + // each entry is 4B and [sample count][sample duration] (since durations can vary) + count = ngx_mp4_get_32value(stts_entry->count); + duration = ngx_mp4_get_32value(stts_entry->duration); + + for (j = 0; j < count; j++) { + sample_num++; + + // search STSS sync sample entries to see if this sample is a keyframe + is_keyframe = (trak->sync_samples_entries ? 0 : 1); + for (n = 0; n < trak->sync_samples_entries; n++) { + // each one of this these are a video sample number keyframe + sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4)); + if (sample_keyframe == sample_num) { + is_keyframe = 1; + break; + } + if (sample_keyframe > sample_num) { + break; + } + } + + seconds = sample_time * 1000 / trak->timescale; + sample_time += duration; + + if (seconds > mp4->start) { + goto found; + } + + if (is_keyframe) { + start_seconds_closest_keyframe = seconds; + exact->speedup_samples = 0; + } else { + exact->speedup_samples++; + } + } + + stts_entry++; + } + + found: + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact_video_adjustment() new keyframe start: %d, speedup first %d samples", + start_seconds_closest_keyframe, + exact->speedup_samples); + + // NOTE: begin 1 start position before keyframe to ensure first video frame emitted is always + // a keyframe + exact->speedup_seconds = mp4->start - start_seconds_closest_keyframe + - (start_seconds_closest_keyframe ? 1 : 0); +} + static ngx_int_t ngx_http_mp4_read_stts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) @@ -2164,10 +2267,14 @@ ngx_buf_t *data; ngx_uint_t start_sample, entries, start_sec; ngx_mp4_stts_entry_t *entry, *end; + ngx_mp4_exact_t exact; if (start) { start_sec = mp4->start; + exact_video_adjustment(mp4, trak, &exact); + start_sec -= exact.speedup_seconds; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "mp4 stts crop start_time:%ui", start_sec); @@ -2230,6 +2337,42 @@ if (start) { ngx_mp4_set_32value(entry->count, count - rest); + + if (exact.speedup_samples) { + // We're going to prepend an entry with duration=1 for the frames we want to "not see". + // MOST of the time, we're taking a single element entry array and making it two. + uint32_t current_count = ngx_mp4_get_32value(entry->count); + ngx_mp4_stts_entry_t* entries_array = ngx_palloc(mp4->request->pool, + (1 + entries) * sizeof(ngx_mp4_stts_entry_t)); + if (entries_array == NULL) { + return NGX_ERROR; + } + ngx_copy(&(entries_array[1]), entry, entries * sizeof(ngx_mp4_stts_entry_t)); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split in 2 video STTS entry from count:%d", current_count); + + if (current_count <= exact.speedup_samples) + return NGX_ERROR; + + entry = &(entries_array[1]); + ngx_mp4_set_32value(entry->count, current_count - exact.speedup_samples); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split new[1]: count:%d duration:%d", + ngx_mp4_get_32value(entry->count), + ngx_mp4_get_32value(entry->duration)); + entry--; + ngx_mp4_set_32value(entry->count, exact.speedup_samples); + ngx_mp4_set_32value(entry->duration, 1); + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split new[0]: count:%d duration:1", + ngx_mp4_get_32value(entry->count)); + + data->last = (u_char *) (entry + 2); + + entries++; + } + data->pos = (u_char *) entry; trak->time_to_sample_entries = entries; trak->start_sample = start_sample; From ecotoper at redhat.com Thu Jun 3 08:12:50 2021 From: ecotoper at redhat.com (Eloy Coto Pereiro) Date: Thu, 3 Jun 2021 10:12:50 +0200 Subject: Proxy_no_cache question Message-ID: Hi, I was checking how proxy_no_cache is handled, and I want to open a discussion here. When it is set to off, should the ngx_http_upstream_cache[0] call at all? I mean, why start the cache options when it's not enabled at all? The primary use case is when the cache is disabled, and things regarding cache are still in there like proxy_cache_convert_head, etc.. I have a small patch about what I'm trying to do. Is it something that you will accept at all? [0] https://github.com/nginx/nginx/blob/5eadaf69e394c030056e4190d86dae0262f8617c/src/http/ngx_http_upstream.c#L823 Patch(Not complete at all) diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index 61fad5ca..1217767d 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -566,42 +566,55 @@ ngx_http_upstream_init_request(ngx_http_request_t *r) #if (NGX_HTTP_CACHE) if (u->conf->cache) { - ngx_int_t rc; - rc = ngx_http_upstream_cache(r, u); + switch (ngx_http_test_predicates(r, u->conf->no_cache)) { - if (rc == NGX_BUSY) { - r->write_event_handler = ngx_http_upstream_init_request; - return; - } + case NGX_ERROR: + return; - r->write_event_handler = ngx_http_request_empty_handler; + case NGX_DECLINED: + /* No need to configure cache at all */ + break; + default: {/* NGX_OK */ + ngx_int_t rc; + rc = ngx_http_upstream_cache(r, u); - if (rc == NGX_ERROR) { - ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); - return; - } + if (rc == NGX_BUSY) { + r->write_event_handler = ngx_http_upstream_init_request; + return; + } - if (rc == NGX_OK) { - rc = ngx_http_upstream_cache_send(r, u); + r->write_event_handler = ngx_http_request_empty_handler; - if (rc == NGX_DONE) { - return; - } + if (rc == NGX_ERROR) { + ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); + return; + } - if (rc == NGX_HTTP_UPSTREAM_INVALID_HEADER) { - rc = NGX_DECLINED; - r->cached = 0; - u->buffer.start = NULL; - u->cache_status = NGX_HTTP_CACHE_MISS; - u->request_sent = 1; - } - } + if (rc == NGX_OK) { + rc = ngx_http_upstream_cache_send(r, u); + + if (rc == NGX_DONE) { + return; + } + + if (rc == NGX_HTTP_UPSTREAM_INVALID_HEADER) { + rc = NGX_DECLINED; + r->cached = 0; + u->buffer.start = NULL; + u->cache_status = NGX_HTTP_CACHE_MISS; + u->request_sent = 1; + } + } + + if (rc != NGX_DECLINED) { + ngx_http_finalize_request(r, rc); + return; + } + break; + } + } - if (rc != NGX_DECLINED) { - ngx_http_finalize_request(r, rc); - return; - } } #endif -------------- next part -------------- An HTML attachment was scrubbed... URL: From mdounin at mdounin.ru Thu Jun 3 13:23:30 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 3 Jun 2021 16:23:30 +0300 Subject: Proxy_no_cache question In-Reply-To: References: Message-ID: Hello! On Thu, Jun 03, 2021 at 10:12:50AM +0200, Eloy Coto Pereiro wrote: > I was checking how proxy_no_cache is handled, and I want to open a > discussion > here. When it is set to off, should the ngx_http_upstream_cache[0] call at > all? I > mean, why start the cache options when it's not enabled at all? The "proxy_no_cache on;" doesn't mean that the cache is not enabled at all. Rather, it means that a particular response should not be saved to cache for some reason. Whether or not it will be possible to save the response might not be known till the response headers are received. If you want to completely disable caching, consider using "proxy_cache off;". -- Maxim Dounin http://mdounin.ru/ From tracey at archive.org Thu Jun 3 19:54:49 2021 From: tracey at archive.org (Tracey Jaquith) Date: Thu, 03 Jun 2021 19:54:49 +0000 Subject: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes Message-ID: <5da9c62fa61016600f2c.1622750089@vm-home1.archive.org> # HG changeset patch # User Tracey Jaquith # Date 1622678642 0 # Thu Jun 03 00:04:02 2021 +0000 # Node ID 5da9c62fa61016600f2c59982ae184e2811be427 # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 Add optional "&exact=1" CGI arg to show video between keyframes. archive.org has been using mod_h264_streaming with an "&exact=1" patch from me since 2013. We just moved to nginx mp4 module and are using this patch. The technique is to find the keyframe just before the desired "start" time, and send that down the wire so video playback can start immediately. Next calculate how many video samples are between the keyframe and desired "start" time and update the STTS atom where those samples move the duration from (typically) 1001 to 1. This way, initial unwanted video frames play at ~1/30,000s -- so visually the video & audio start playing immediately. You can see an example before/after here (nginx binary built with mp4 module + patch): https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30 https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30&exact=1 Tested on linux and macosx. I realize two var declarations should style-wise get moved "up" in the lower hooked in method. However, to minimize the changeset/patch, I figured we should at least start here and see what you folks think. (this is me: https://github.com/traceypooh ) diff -r 5f765427c17a -r 5da9c62fa610 src/http/modules/ngx_http_mp4_module.c --- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300 +++ b/src/http/modules/ngx_http_mp4_module.c Thu Jun 03 00:04:02 2021 +0000 @@ -2045,6 +2045,109 @@ u_char duration[4]; } ngx_mp4_stts_entry_t; +typedef struct { + uint32_t speedup_samples; + ngx_uint_t speedup_seconds; +} ngx_mp4_exact_t; + +static void +exact_video_adjustment(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_mp4_exact_t *exact) +{ + // will parse STTS -- time-to-sample atom + ngx_str_t value; + ngx_buf_t *stts_data; + ngx_buf_t *atom; + ngx_mp4_stts_entry_t *stts_entry, *stts_end; + uint32_t count, duration, j, n, sample_keyframe, sample_num; + uint64_t sample_time, seconds, start_seconds_closest_keyframe; + uint8_t is_keyframe; + + exact->speedup_samples = 0; + exact->speedup_seconds = 0; + + // if '&exact=1' CGI arg isn't present, do nothing + if (!(ngx_http_arg(mp4->request, (u_char *) "exact", 5, &value) == NGX_OK)) { + return; + } + + if (!trak->sync_samples_entries) { + // Highly unlikely video STSS got parsed and _every_ sample is a keyframe. + // However, if the case, we don't need to adjust the video at all. + return; + } + + // check HDLR atom to see if this trak is video or audio + atom = trak->out[NGX_HTTP_MP4_HDLR_ATOM].buf; + // 'vide' or 'soun' + if (!(atom->pos[16] == 'v' && + atom->pos[17] == 'i' && + atom->pos[18] == 'd' && + atom->pos[19] == 'e')) { + return; // do nothing if not video + } + + + stts_data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf; + stts_entry = (ngx_mp4_stts_entry_t *) stts_data->pos; + stts_end = (ngx_mp4_stts_entry_t *) stts_data->last; + + sample_num = 0; // they start at one + sample_time = 0; + start_seconds_closest_keyframe = 0; + while (stts_entry < stts_end) { + // STTS === time-to-sample atom + // each entry is 4B and [sample count][sample duration] (since durations can vary) + count = ngx_mp4_get_32value(stts_entry->count); + duration = ngx_mp4_get_32value(stts_entry->duration); + + for (j = 0; j < count; j++) { + sample_num++; + + // search STSS sync sample entries to see if this sample is a keyframe + is_keyframe = (trak->sync_samples_entries ? 0 : 1); + for (n = 0; n < trak->sync_samples_entries; n++) { + // each one of this these are a video sample number keyframe + sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4)); + if (sample_keyframe == sample_num) { + is_keyframe = 1; + break; + } + if (sample_keyframe > sample_num) { + break; + } + } + + seconds = sample_time * 1000 / trak->timescale; + sample_time += duration; + + if (seconds > mp4->start) { + goto found; + } + + if (is_keyframe) { + start_seconds_closest_keyframe = seconds; + exact->speedup_samples = 0; + } else { + exact->speedup_samples++; + } + } + + stts_entry++; + } + + found: + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact_video_adjustment() new keyframe start: %d, speedup first %d samples", + start_seconds_closest_keyframe, + exact->speedup_samples); + + // NOTE: begin 1 start position before keyframe to ensure first video frame emitted is always + // a keyframe + exact->speedup_seconds = mp4->start - start_seconds_closest_keyframe + - (start_seconds_closest_keyframe ? 1 : 0); +} + static ngx_int_t ngx_http_mp4_read_stts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) @@ -2164,10 +2267,14 @@ ngx_buf_t *data; ngx_uint_t start_sample, entries, start_sec; ngx_mp4_stts_entry_t *entry, *end; + ngx_mp4_exact_t exact; if (start) { start_sec = mp4->start; + exact_video_adjustment(mp4, trak, &exact); + start_sec -= exact.speedup_seconds; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "mp4 stts crop start_time:%ui", start_sec); @@ -2230,6 +2337,42 @@ if (start) { ngx_mp4_set_32value(entry->count, count - rest); + + if (exact.speedup_samples) { + // We're going to prepend an entry with duration=1 for the frames we want to "not see". + // MOST of the time, we're taking a single element entry array and making it two. + uint32_t current_count = ngx_mp4_get_32value(entry->count); + ngx_mp4_stts_entry_t* entries_array = ngx_palloc(mp4->request->pool, + (1 + entries) * sizeof(ngx_mp4_stts_entry_t)); + if (entries_array == NULL) { + return NGX_ERROR; + } + ngx_copy(&(entries_array[1]), entry, entries * sizeof(ngx_mp4_stts_entry_t)); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split in 2 video STTS entry from count:%d", current_count); + + if (current_count <= exact.speedup_samples) + return NGX_ERROR; + + entry = &(entries_array[1]); + ngx_mp4_set_32value(entry->count, current_count - exact.speedup_samples); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split new[1]: count:%d duration:%d", + ngx_mp4_get_32value(entry->count), + ngx_mp4_get_32value(entry->duration)); + entry--; + ngx_mp4_set_32value(entry->count, exact.speedup_samples); + ngx_mp4_set_32value(entry->duration, 1); + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split new[0]: count:%d duration:1", + ngx_mp4_get_32value(entry->count)); + + data->last = (u_char *) (entry + 2); + + entries++; + } + data->pos = (u_char *) entry; trak->time_to_sample_entries = entries; trak->start_sample = start_sample; From Siddesh.Sangodkar at ibm.com Fri Jun 4 04:53:37 2021 From: Siddesh.Sangodkar at ibm.com (Siddesh Sangodkar) Date: Fri, 4 Jun 2021 10:23:37 +0530 Subject: Add s390x support for NGINX linux packages Message-ID: Hi All, As seen from downloads package?here. the Apt repo/rpms are not yet available for s390x architecture. Wanted to know if there is any work going on/planned to provide nginx in form of linux packages for s390x architecture? Any pointers will be helpful. Thank you in advance. Regards. Siddesh Sangodkar -------------- next part -------------- An HTML attachment was scrubbed... URL: From thresh at nginx.com Fri Jun 4 08:39:14 2021 From: thresh at nginx.com (Konstantin Pavlov) Date: Fri, 4 Jun 2021 11:39:14 +0300 Subject: Add s390x support for NGINX linux packages In-Reply-To: References: Message-ID: Hi Siddesh, 04.06.2021 07:53, Siddesh Sangodkar wrote: > Hi All, > > As seen from downloads > package?here > . the Apt repo/rpms are not yet > available for s390x architecture. > Wanted to know if there is any work going on/planned to provide nginx in > form of linux packages for s390x architecture? > Any pointers will be helpful. There are no plans for s390x to be a supported architecture for nginx.org packages. However the packaging sources are available as per http://nginx.org/en/linux_packages.html#sourcepackages and it should be possible to build those manually if you need to - e.g. it's known that Debian and Alpine packages require no modifications to build and work on that architecture. Have a good day, -- Konstantin Pavlov https://www.nginx.com/ From mdounin at mdounin.ru Fri Jun 4 17:38:09 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Fri, 4 Jun 2021 20:38:09 +0300 Subject: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes In-Reply-To: <5da9c62fa61016600f2c.1622750089@vm-home1.archive.org> References: <5da9c62fa61016600f2c.1622750089@vm-home1.archive.org> Message-ID: Hello! On Thu, Jun 03, 2021 at 07:54:49PM +0000, Tracey Jaquith wrote: > # HG changeset patch > # User Tracey Jaquith > # Date 1622678642 0 > # Thu Jun 03 00:04:02 2021 +0000 > # Node ID 5da9c62fa61016600f2c59982ae184e2811be427 > # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 > Add optional "&exact=1" CGI arg to show video between keyframes. > > archive.org has been using mod_h264_streaming with an "&exact=1" patch from me since 2013. > We just moved to nginx mp4 module and are using this patch. > The technique is to find the keyframe just before the desired "start" time, and send > that down the wire so video playback can start immediately. > Next calculate how many video samples are between the keyframe and desired "start" time > and update the STTS atom where those samples move the duration from (typically) 1001 to 1. > This way, initial unwanted video frames play at ~1/30,000s -- so visually the > video & audio start playing immediately. So, effectively, this makes invalid streaming requests (not starting at a keyframe) to return the same data as valid ones (starting at a keyframe). A properly written player should be able to achieve the same result by making correct streaming requests. Any specific reasons to do this within nginx instead of fixing the player? Or, given the fact that Flash is effectively dead, by simply using HTML5 video with appropriate seeking? -- Maxim Dounin http://mdounin.ru/ From tracey at archive.org Sat Jun 5 02:18:55 2021 From: tracey at archive.org (Tracey Jaquith) Date: Fri, 4 Jun 2021 19:18:55 -0700 Subject: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes In-Reply-To: References: <5da9c62fa61016600f2c.1622750089@vm-home1.archive.org> Message-ID: <3A5F7B9F-4693-41C2-8DB8-C2E1DD00AC55@archive.org> Greetings and hello Maxim, Thanks for the reply and excellent question and points. So.. yes, certainly enough information (worst case scenario, the entire moov atom) could be sent down to the client to figure out what best keyframes to jump to (and then range request it, etc.) However: That?s potentially a lot of data in the full moov atom to go down the wire, just to know where to start playing from, for a single .mp4 file. For example, for a 2-3 hour HD TV show, that can easily exceed 10MB of moov atom that needs to head to the client before it can even start playing a single first video frame. Compared to, say, HLS/DASH where playback can begin very quickly. We?re relying on the server side ?clipping? for our TV News Archive ( https://tv.archive.org ) to restrict the portion of the videos we permit our users to view. So we?re always only permitting streaming of relatively short clips (~15 to 120s) since we consider this as viewing samples from a larger TV news show (ie: we aren?t a VOD type Archive ? we?re serving search results and trying to help researchers, etc.) It seems to me that either way, the value in the mp4 module is primarily in that server-side ?magic? of opening and rewriting the moov atom, and then losslessly sending the A/V packets down to the client. It saves a huge amount of bandwidth for smaller clips, seeking, and even jumping mid-way into a film or TV show ? for archives and playback services that have more constraints on internal storage. The Internet Archive, in general, only makes a single .mp4 web-friendly video file from uploaded materials. We don?t have the storage to make 2-5 streaming qualities for all our videos, and chunk them into small HLS/DASH pieces with an ideal adaptive/feedback constantly changing quality system (like bigger players like Netflix, YouTube, etc.) I figure the ?begin video immediately? patch (this patch) could be useful to others, writ-large. It certainly has been of keen interest to our TV viewers and researchers and data visualization folks. I suspect that most of the users of this mp4 module might find the ?being video immediately? (perceptively) a nice optional improvement. I suppose a middle ground (but would also be a change/patch) could be to always start at the keyframe before the wanted start-point ? and have the client, with JS, always seek to the desired start point. However, it?s definitely nice magic to request a specific start/end point, and get exactly that, down to 1/10th of a second, with no need for client JS to seek. They can play directly in all browsers and mobile as a legit .mp4 file that?s been efficiently served up just for you, on demand, losslessly. Magic! :) And as a sidenote, I must confess, that the 3500 lines of C code (in a single file!), would normally be a run-away-screaming situation for me. However, never in my life (and early start with C) have I seen such well-structured and clever C code like that. I was pretty speechless exploring it! Thanks for the consideration, -Tracey > On Jun 4, 2021, at 10:38 AM, Maxim Dounin wrote: > > Hello! > > On Thu, Jun 03, 2021 at 07:54:49PM +0000, Tracey Jaquith wrote: > >> # HG changeset patch >> # User Tracey Jaquith >> # Date 1622678642 0 >> # Thu Jun 03 00:04:02 2021 +0000 >> # Node ID 5da9c62fa61016600f2c59982ae184e2811be427 >> # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 >> Add optional "&exact=1" CGI arg to show video between keyframes. >> >> archive.org has been using mod_h264_streaming with an "&exact=1" patch from me since 2013. >> We just moved to nginx mp4 module and are using this patch. >> The technique is to find the keyframe just before the desired "start" time, and send >> that down the wire so video playback can start immediately. >> Next calculate how many video samples are between the keyframe and desired "start" time >> and update the STTS atom where those samples move the duration from (typically) 1001 to 1. >> This way, initial unwanted video frames play at ~1/30,000s -- so visually the >> video & audio start playing immediately. > > So, effectively, this makes invalid streaming requests (not > starting at a keyframe) to return the same data as valid ones > (starting at a keyframe). A properly written player should be > able to achieve the same result by making correct streaming > requests. > > Any specific reasons to do this within nginx instead of fixing the > player? Or, given the fact that Flash is effectively dead, by > simply using HTML5 video with appropriate seeking? > > -- > Maxim Dounin > http://mdounin.ru/ > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > http://mailman.nginx.org/mailman/listinfo/nginx-devel @tracey_pooh TV Architect https://archive.org/tv -------------- next part -------------- An HTML attachment was scrubbed... URL: From naveenamresh300496 at gmail.com Sat Jun 5 04:47:09 2021 From: naveenamresh300496 at gmail.com (Naveen Amresh) Date: Sat, 5 Jun 2021 10:17:09 +0530 Subject: REQ: NGINX LDAP Password Masking (nginx.conf) Message-ID: Hi Nginx Support Team, I would like to get assistance in Nginx LDAP bindn password masking in nginx.conf file. Thanks & Regards - Naveen -------------- next part -------------- An HTML attachment was scrubbed... URL: From mdounin at mdounin.ru Sun Jun 6 15:52:00 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Sun, 6 Jun 2021 18:52:00 +0300 Subject: REQ: NGINX LDAP Password Masking (nginx.conf) In-Reply-To: References: Message-ID: Hello! On Sat, Jun 05, 2021 at 10:17:09AM +0530, Naveen Amresh wrote: > Hi Nginx Support Team, > > I would like to get assistance in Nginx LDAP bindn password masking in > nginx.conf file. This is not a support mailing list, but rather a mailing list for nginx developers. Please avoid asking user-level questions here. Thank you. -- Maxim Dounin http://mdounin.ru/ From xeioex at nginx.com Mon Jun 7 19:15:39 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Mon, 07 Jun 2021 19:15:39 +0000 Subject: [njs] Implemented RegExp getters according to the specification. Message-ID: details: https://hg.nginx.org/njs/rev/02444445df29 branches: changeset: 1650:02444445df29 user: Dmitry Volyntsev date: Sat Jun 05 11:58:00 2021 +0000 description: Implemented RegExp getters according to the specification. diffstat: src/njs_regexp.c | 269 ++++++++++++++++++++++++++++++++++++++-------- src/njs_regexp_pattern.h | 5 +- src/njs_value.c | 2 + src/njs_value.h | 1 + src/test/njs_unit_test.c | 13 +- 5 files changed, 235 insertions(+), 55 deletions(-) diffs (460 lines): diff -r 687d9eacbe33 -r 02444445df29 src/njs_regexp.c --- a/src/njs_regexp.c Wed Jun 02 14:10:03 2021 +0000 +++ b/src/njs_regexp.c Sat Jun 05 11:58:00 2021 +0000 @@ -17,9 +17,8 @@ struct njs_regexp_group_s { static void *njs_regexp_malloc(size_t size, void *memory_data); static void njs_regexp_free(void *p, void *memory_data); -static njs_int_t njs_regexp_prototype_source(njs_vm_t *vm, - njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval, - njs_value_t *retval); +static njs_int_t njs_regexp_prototype_source(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t unused); static int njs_regexp_pattern_compile(njs_vm_t *vm, njs_regex_t *regex, u_char *source, int options); static u_char *njs_regexp_compile_trace_handler(njs_trace_t *trace, @@ -108,11 +107,13 @@ njs_regexp_constructor(njs_vm_t *vm, njs pattern = njs_arg(args, nargs, 1); if (njs_is_regexp(pattern)) { - ret = njs_regexp_prototype_source(vm, NULL, pattern, NULL, &source); + ret = njs_regexp_prototype_source(vm, pattern, 1, 0); if (njs_slow_path(ret != NJS_OK)) { return ret; } + source = vm->retval; + re_flags = njs_regexp_value_flags(vm, pattern); pattern = &source; @@ -634,64 +635,141 @@ njs_regexp_prototype_last_index(njs_vm_t static njs_int_t -njs_regexp_prototype_global(njs_vm_t *vm, njs_object_prop_t *prop, - njs_value_t *value, njs_value_t *setval, njs_value_t *retval) +njs_regexp_prototype_flags(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t unused) { - njs_regexp_pattern_t *pattern; + u_char *p; + njs_int_t ret; + njs_value_t *this, value; + u_char dst[3]; + + static const njs_value_t string_global = njs_string("global"); + static const njs_value_t string_ignore_case = njs_string("ignoreCase"); + static const njs_value_t string_multiline = njs_string("multiline"); + + this = njs_argument(args, 0); + if (njs_slow_path(!njs_is_object(this))) { + njs_type_error(vm, "\"this\" argument is not an object"); + return NJS_ERROR; + } + + p = &dst[0]; + + ret = njs_value_property(vm, this, njs_value_arg(&string_global), + &value); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } - pattern = njs_regexp_pattern(value); - *retval = pattern->global ? njs_value_true : njs_value_false; - njs_release(vm, value); + if (njs_bool(&value)) { + *p++ = 'g'; + } + + ret = njs_value_property(vm, this, njs_value_arg(&string_ignore_case), + &value); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } - return NJS_OK; + if (njs_bool(&value)) { + *p++ = 'i'; + } + + ret = njs_value_property(vm, this, njs_value_arg(&string_multiline), + &value); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + if (njs_bool(&value)) { + *p++ = 'm'; + } + + return njs_string_new(vm, &vm->retval, dst, p - dst, p - dst); } static njs_int_t -njs_regexp_prototype_ignore_case(njs_vm_t *vm, njs_object_prop_t *prop, - njs_value_t *value, njs_value_t *setval, njs_value_t *retval) +njs_regexp_prototype_flag(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t flag) { + unsigned yn; + njs_value_t *this; njs_regexp_pattern_t *pattern; - pattern = njs_regexp_pattern(value); - *retval = pattern->ignore_case ? njs_value_true : njs_value_false; - njs_release(vm, value); + this = njs_argument(args, 0); + if (njs_slow_path(!njs_is_object(this))) { + njs_type_error(vm, "\"this\" argument is not an object"); + return NJS_ERROR; + } + + if (njs_slow_path(!njs_is_regexp(this))) { + if (njs_object(this) == &vm->prototypes[NJS_OBJ_TYPE_REGEXP].object) { + njs_set_undefined(&vm->retval); + return NJS_OK; + } + + njs_type_error(vm, "\"this\" argument is not a regexp"); + return NJS_ERROR; + } + + pattern = njs_regexp_pattern(this); + + switch (flag) { + case NJS_REGEXP_GLOBAL: + yn = pattern->global; + break; + + case NJS_REGEXP_IGNORE_CASE: + yn = pattern->ignore_case; + break; + + case NJS_REGEXP_MULTILINE: + default: + yn = pattern->multiline; + break; + } + + njs_set_boolean(&vm->retval, yn); return NJS_OK; } static njs_int_t -njs_regexp_prototype_multiline(njs_vm_t *vm, njs_object_prop_t *prop, - njs_value_t *value, njs_value_t *setval, njs_value_t *retval) -{ - njs_regexp_pattern_t *pattern; - - pattern = njs_regexp_pattern(value); - *retval = pattern->multiline ? njs_value_true : njs_value_false; - njs_release(vm, value); - - return NJS_OK; -} - - -static njs_int_t -njs_regexp_prototype_source(njs_vm_t *vm, njs_object_prop_t *prop, - njs_value_t *value, njs_value_t *setval, njs_value_t *retval) +njs_regexp_prototype_source(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t unused) { u_char *source; int32_t length; uint32_t size; + njs_value_t *this; njs_regexp_pattern_t *pattern; - pattern = njs_regexp_pattern(value); + this = njs_argument(args, 0); + if (njs_slow_path(!njs_is_object(this))) { + njs_type_error(vm, "\"this\" argument is not an object"); + return NJS_ERROR; + } + + if (njs_slow_path(!njs_is_regexp(this))) { + if (njs_object(this) == &vm->prototypes[NJS_OBJ_TYPE_REGEXP].object) { + vm->retval = njs_string_empty_regexp; + return NJS_OK; + } + + njs_type_error(vm, "\"this\" argument is not a regexp"); + return NJS_ERROR; + } + + pattern = njs_regexp_pattern(this); /* Skip starting "/". */ source = pattern->source + 1; size = njs_strlen(source) - pattern->flags; length = njs_utf8_length(source, size); - return njs_regexp_string_create(vm, retval, source, size, length); + return njs_regexp_string_create(vm, &vm->retval, source, size, length); } @@ -699,13 +777,67 @@ static njs_int_t njs_regexp_prototype_to_string(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused) { - if (njs_is_regexp(njs_arg(args, nargs, 0))) { - return njs_regexp_to_string(vm, &vm->retval, &args[0]); + u_char *p; + size_t size, length; + njs_int_t ret; + njs_value_t *r, source, flags; + njs_string_prop_t source_string, flags_string; + + static const njs_value_t string_source = njs_string("source"); + static const njs_value_t string_flags = njs_string("flags"); + + r = njs_argument(args, 0); + + if (njs_slow_path(!njs_is_object(r))) { + njs_type_error(vm, "\"this\" argument is not an object"); + return NJS_ERROR; + } + + ret = njs_value_property(vm, r, njs_value_arg(&string_source), + &source); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + ret = njs_value_to_string(vm, &source, &source); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; } - njs_type_error(vm, "\"this\" argument is not a regexp"); + ret = njs_value_property(vm, r, njs_value_arg(&string_flags), + &flags); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + ret = njs_value_to_string(vm, &flags, &flags); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + (void) njs_string_prop(&source_string, &source); + (void) njs_string_prop(&flags_string, &flags); + + size = source_string.size + flags_string.size + njs_length("//"); + length = source_string.length + flags_string.length + njs_length("//"); - return NJS_ERROR; + if (njs_is_byte_string(&source_string) + || njs_is_byte_string(&flags_string)) + { + length = 0; + } + + p = njs_string_alloc(vm, &vm->retval, size, length); + if (njs_slow_path(p == NULL)) { + return NJS_ERROR; + } + + *p++ = '/'; + p = njs_cpymem(p, source_string.start, source_string.size); + *p++ = '/'; + memcpy(p, flags_string.start, flags_string.size); + + return NJS_OK; } @@ -713,7 +845,7 @@ njs_int_t njs_regexp_to_string(njs_vm_t *vm, njs_value_t *retval, const njs_value_t *value) { - u_char *source; + u_char *p, *source; int32_t length; uint32_t size; njs_regexp_pattern_t *pattern; @@ -724,7 +856,16 @@ njs_regexp_to_string(njs_vm_t *vm, njs_v size = njs_strlen(source); length = njs_utf8_length(source, size); - return njs_regexp_string_create(vm, retval, source, size, length); + length = (length >= 0) ? length: 0; + + p = njs_string_alloc(vm, retval, size, length); + if (njs_slow_path(p == NULL)) { + return NJS_ERROR; + } + + p = njs_cpymem(p, source, size); + + return NJS_OK; } @@ -1522,31 +1663,61 @@ static const njs_object_prop_t njs_rege }, { - .type = NJS_PROPERTY_HANDLER, + .type = NJS_PROPERTY, + .name = njs_string("flags"), + .value = njs_value(NJS_INVALID, 1, NAN), + .getter = njs_native_function(njs_regexp_prototype_flags, 0), + .setter = njs_value(NJS_UNDEFINED, 0, NAN), + .writable = NJS_ATTRIBUTE_UNSET, + .configurable = 1, + .enumerable = 0, + }, + + { + .type = NJS_PROPERTY, .name = njs_string("global"), - .value = njs_prop_handler(njs_regexp_prototype_global), + .value = njs_value(NJS_INVALID, 1, NAN), + .getter = njs_native_function2(njs_regexp_prototype_flag, 0, + NJS_REGEXP_GLOBAL), + .setter = njs_value(NJS_UNDEFINED, 0, NAN), + .writable = NJS_ATTRIBUTE_UNSET, .configurable = 1, + .enumerable = 0, }, { - .type = NJS_PROPERTY_HANDLER, + .type = NJS_PROPERTY, .name = njs_string("ignoreCase"), - .value = njs_prop_handler(njs_regexp_prototype_ignore_case), + .value = njs_value(NJS_INVALID, 1, NAN), + .getter = njs_native_function2(njs_regexp_prototype_flag, 0, + NJS_REGEXP_IGNORE_CASE), + .setter = njs_value(NJS_UNDEFINED, 0, NAN), + .writable = NJS_ATTRIBUTE_UNSET, .configurable = 1, + .enumerable = 0, }, { - .type = NJS_PROPERTY_HANDLER, + .type = NJS_PROPERTY, .name = njs_string("multiline"), - .value = njs_prop_handler(njs_regexp_prototype_multiline), + .value = njs_value(NJS_INVALID, 1, NAN), + .getter = njs_native_function2(njs_regexp_prototype_flag, 0, + NJS_REGEXP_MULTILINE), + .setter = njs_value(NJS_UNDEFINED, 0, NAN), + .writable = NJS_ATTRIBUTE_UNSET, .configurable = 1, + .enumerable = 0, }, { - .type = NJS_PROPERTY_HANDLER, + .type = NJS_PROPERTY, .name = njs_string("source"), - .value = njs_prop_handler(njs_regexp_prototype_source), + .value = njs_value(NJS_INVALID, 1, NAN), + .getter = njs_native_function(njs_regexp_prototype_source, 0), + .setter = njs_value(NJS_UNDEFINED, 0, NAN), + .writable = NJS_ATTRIBUTE_UNSET, .configurable = 1, + .enumerable = 0, }, { @@ -1610,5 +1781,5 @@ const njs_object_type_init_t njs_regexp .constructor = njs_native_ctor(njs_regexp_constructor, 2, 0), .constructor_props = &njs_regexp_constructor_init, .prototype_props = &njs_regexp_prototype_init, - .prototype_value = { .object = { .type = NJS_REGEXP } }, + .prototype_value = { .object = { .type = NJS_OBJECT } }, }; diff -r 687d9eacbe33 -r 02444445df29 src/njs_regexp_pattern.h --- a/src/njs_regexp_pattern.h Wed Jun 02 14:10:03 2021 +0000 +++ b/src/njs_regexp_pattern.h Sat Jun 05 11:58:00 2021 +0000 @@ -21,8 +21,9 @@ struct njs_regexp_pattern_s { njs_regex_t regex[2]; /* - * A pattern source is used by RegExp.toString() method and - * RegExp.source property. So it is is stored in form "/pattern/flags" + * A pattern source is used by RegExp.prototype.toString() method and + * RegExp.prototype.source and RegExp.prototype.flags accessor properties. + * So it is is stored in form "/pattern/flags" * and as zero-terminated C string but not as value, because retrieving * it is very seldom operation. To get just a pattern string for * RegExp.source property a length of flags part "/flags" is stored diff -r 687d9eacbe33 -r 02444445df29 src/njs_value.c --- a/src/njs_value.c Wed Jun 02 14:10:03 2021 +0000 +++ b/src/njs_value.c Sat Jun 05 11:58:00 2021 +0000 @@ -30,6 +30,8 @@ const njs_value_t njs_value_nan = const njs_value_t njs_value_invalid = njs_value(NJS_INVALID, 0, 0.0); const njs_value_t njs_string_empty = njs_string(""); +const njs_value_t njs_string_empty_regexp = + njs_string("(?:)"); const njs_value_t njs_string_comma = njs_string(","); const njs_value_t njs_string_null = njs_string("null"); const njs_value_t njs_string_undefined = njs_string("undefined"); diff -r 687d9eacbe33 -r 02444445df29 src/njs_value.h --- a/src/njs_value.h Wed Jun 02 14:10:03 2021 +0000 +++ b/src/njs_value.h Sat Jun 05 11:58:00 2021 +0000 @@ -802,6 +802,7 @@ extern const njs_value_t njs_value_nan; extern const njs_value_t njs_value_invalid; extern const njs_value_t njs_string_empty; +extern const njs_value_t njs_string_empty_regexp; extern const njs_value_t njs_string_comma; extern const njs_value_t njs_string_null; extern const njs_value_t njs_string_undefined; diff -r 687d9eacbe33 -r 02444445df29 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Wed Jun 02 14:10:03 2021 +0000 +++ b/src/test/njs_unit_test.c Sat Jun 05 11:58:00 2021 +0000 @@ -10846,6 +10846,11 @@ static njs_unit_test_t njs_test[] = { njs_str("var r = (/^.+$/mg); [r.global, r.multiline, r.ignoreCase]"), njs_str("true,true,false") }, + { njs_str("['global', 'ignoreCase', 'multiline']" + ".map(v => Object.getOwnPropertyDescriptor(RegExp.prototype, v))" + ".every(desc => (typeof desc.get === 'function' && typeof desc.set === 'undefined'))"), + njs_str("true") }, + { njs_str("var r = /./; r"), njs_str("/./") }, @@ -10855,8 +10860,8 @@ static njs_unit_test_t njs_test[] = { njs_str("var r = new RegExp('.'); r"), njs_str("/./") }, - { njs_str("var r = new RegExp('.', 'ig'); r"), - njs_str("/./gi") }, + { njs_str("var r = new RegExp('.', 'igm'); r"), + njs_str("/./gim") }, { njs_str("var r = new RegExp('abc'); r.test('00abc11')"), njs_str("true") }, @@ -10922,7 +10927,7 @@ static njs_unit_test_t njs_test[] = njs_str("SyntaxError: pcre_compile(\"\\\") failed: \\ at end of pattern") }, { njs_str("[0].map(RegExp().toString)"), - njs_str("TypeError: \"this\" argument is not a regexp") }, + njs_str("TypeError: \"this\" argument is not an object") }, /* Non-standard ECMA-262 features. */ @@ -12986,7 +12991,7 @@ static njs_unit_test_t njs_test[] = njs_str("true") }, { njs_str("Object.prototype.toString.call(RegExp.prototype)"), - njs_str("[object RegExp]") }, + njs_str("[object Object]") }, { njs_str("RegExp.prototype"), njs_str("/(?:)/") }, From xeioex at nginx.com Mon Jun 7 19:16:13 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Mon, 07 Jun 2021 19:16:13 +0000 Subject: [njs] Fixed RegExp.prototype.test() according to the specification. Message-ID: details: https://hg.nginx.org/njs/rev/de64420d0f2b branches: changeset: 1651:de64420d0f2b user: Dmitry Volyntsev date: Sat Jun 05 11:55:08 2021 +0000 description: Fixed RegExp.prototype.test() according to the specification. diffstat: src/njs_regexp.c | 91 +++++++++---------------------------------------------- 1 files changed, 15 insertions(+), 76 deletions(-) diffs (111 lines): diff -r 02444445df29 -r de64420d0f2b src/njs_regexp.c --- a/src/njs_regexp.c Sat Jun 05 11:58:00 2021 +0000 +++ b/src/njs_regexp.c Sat Jun 05 11:55:08 2021 +0000 @@ -873,92 +873,31 @@ static njs_int_t njs_regexp_prototype_test(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused) { - int *captures; - int64_t last_index; - njs_int_t ret, match; - njs_uint_t n; - njs_regex_t *regex; - njs_regexp_t *regexp; - njs_value_t *value, lvalue; - const njs_value_t *retval; - njs_string_prop_t string; - njs_regexp_pattern_t *pattern; - njs_regex_match_data_t *match_data; + njs_int_t ret; + njs_value_t *r, *string, lvalue, retval; - if (!njs_is_regexp(njs_arg(args, nargs, 0))) { - njs_type_error(vm, "\"this\" argument is not a regexp"); + r = njs_argument(args, 0); + + if (njs_slow_path(!njs_is_object(r))) { + njs_type_error(vm, "\"this\" argument is not an object"); return NJS_ERROR; } - retval = &njs_value_false; - - value = njs_lvalue_arg(&lvalue, args, nargs, 1); + string = njs_lvalue_arg(&lvalue, args, nargs, 1); - if (!njs_is_string(value)) { - ret = njs_value_to_string(vm, value, value); - if (njs_slow_path(ret != NJS_OK)) { - return ret; - } + ret = njs_value_to_string(vm, string, string); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; } - (void) njs_string_prop(&string, value); - - n = (string.length != 0); - - regexp = njs_regexp(njs_argument(args, 0)); - pattern = njs_regexp_pattern(&args[0]); - - regex = &pattern->regex[n]; - match_data = vm->single_match_data; - - if (njs_regex_is_valid(regex)) { - if (njs_regex_backrefs(regex) != 0) { - match_data = njs_regex_match_data(regex, vm->regex_context); - if (njs_slow_path(match_data == NULL)) { - njs_memory_error(vm); - return NJS_ERROR; - } - } - - match = njs_regexp_match(vm, regex, string.start, 0, string.size, - match_data); - if (match >= 0) { - retval = &njs_value_true; - - } else if (match != NJS_REGEX_NOMATCH) { - ret = NJS_ERROR; - goto done; - } - - if (pattern->global) { - ret = njs_value_to_length(vm, ®exp->last_index, &last_index); - if (njs_slow_path(ret != NJS_OK)) { - return NJS_ERROR; - } - - if (match >= 0) { - captures = njs_regex_captures(match_data); - last_index += captures[1]; - - } else { - last_index = 0; - } - - njs_set_number(®exp->last_index, last_index); - } + ret = njs_regexp_exec(vm, r, string, &retval); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; } - ret = NJS_OK; - - vm->retval = *retval; - -done: + njs_set_boolean(&vm->retval, !njs_is_null(&retval)); - if (match_data != vm->single_match_data) { - njs_regex_match_data_free(match_data, vm->regex_context); - } - - return ret; + return NJS_OK; } From xeioex at nginx.com Mon Jun 7 19:16:15 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Mon, 07 Jun 2021 19:16:15 +0000 Subject: [njs] Accessing "lastIndex" property according to the spec. Message-ID: details: https://hg.nginx.org/njs/rev/45b9af3f911f branches: changeset: 1652:45b9af3f911f user: Dmitry Volyntsev date: Sat Jun 05 11:55:08 2021 +0000 description: Accessing "lastIndex" property according to the spec. diffstat: src/njs_regexp.c | 47 ++++++++++++++++++++++++++++++++++------------- src/test/njs_unit_test.c | 6 ++++++ 2 files changed, 40 insertions(+), 13 deletions(-) diffs (164 lines): diff -r de64420d0f2b -r 45b9af3f911f src/njs_regexp.c --- a/src/njs_regexp.c Sat Jun 05 11:55:08 2021 +0000 +++ b/src/njs_regexp.c Sat Jun 05 11:55:08 2021 +0000 @@ -25,13 +25,16 @@ static u_char *njs_regexp_compile_trace_ njs_trace_data_t *td, u_char *start); static u_char *njs_regexp_match_trace_handler(njs_trace_t *trace, njs_trace_data_t *td, u_char *start); -static njs_array_t *njs_regexp_exec_result(njs_vm_t *vm, njs_regexp_t *regexp, +static njs_array_t *njs_regexp_exec_result(njs_vm_t *vm, njs_value_t *r, njs_regexp_utf8_t type, njs_string_prop_t *string, njs_regex_match_data_t *data); static njs_int_t njs_regexp_string_create(njs_vm_t *vm, njs_value_t *value, u_char *start, uint32_t size, int32_t length); +const njs_value_t njs_string_lindex = njs_string("lastIndex"); + + njs_int_t njs_regexp_init(njs_vm_t *vm) { @@ -911,6 +914,7 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj size_t length, offset; int64_t last_index; njs_int_t ret; + njs_value_t value; njs_array_t *result; njs_regexp_t *regexp; njs_string_prop_t string; @@ -922,7 +926,13 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj regexp->string = *s; pattern = regexp->pattern; - ret = njs_value_to_length(vm, ®exp->last_index, &last_index); + ret = njs_value_property(vm, r, njs_value_arg(&njs_string_lindex), + &value); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + ret = njs_value_to_length(vm, &value, &last_index); if (njs_slow_path(ret != NJS_OK)) { return NJS_ERROR; } @@ -968,7 +978,7 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj ret = njs_regexp_match(vm, &pattern->regex[type], string.start, offset, string.size, match_data); if (ret >= 0) { - result = njs_regexp_exec_result(vm, regexp, type, &string, match_data); + result = njs_regexp_exec_result(vm, r, type, &string, match_data); if (njs_slow_path(result == NULL)) { return NJS_ERROR; } @@ -986,7 +996,12 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj not_found: if (pattern->global) { - njs_set_number(®exp->last_index, 0); + njs_set_number(&value, 0); + ret = njs_value_property_set(vm, r, njs_value_arg(&njs_string_lindex), + &value); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } } njs_set_null(retval); @@ -996,9 +1011,8 @@ not_found: static njs_array_t * -njs_regexp_exec_result(njs_vm_t *vm, njs_regexp_t *regexp, - njs_regexp_utf8_t type, njs_string_prop_t *string, - njs_regex_match_data_t *match_data) +njs_regexp_exec_result(njs_vm_t *vm, njs_value_t *r, njs_regexp_utf8_t type, + njs_string_prop_t *string, njs_regex_match_data_t *match_data) { int *captures; u_char *start; @@ -1007,8 +1021,9 @@ njs_regexp_exec_result(njs_vm_t *vm, njs njs_int_t ret; njs_uint_t i, n; njs_array_t *array; - njs_value_t name; + njs_value_t name, value; njs_object_t *groups; + njs_regexp_t *regexp; njs_object_prop_t *prop; njs_regexp_group_t *group; njs_lvlhsh_query_t lhq; @@ -1018,6 +1033,7 @@ njs_regexp_exec_result(njs_vm_t *vm, njs static const njs_value_t string_input = njs_string("input"); static const njs_value_t string_groups = njs_string("groups"); + regexp = njs_regexp(r); pattern = regexp->pattern; array = njs_array_alloc(vm, 0, pattern->ncaptures, 0); if (njs_slow_path(array == NULL)) { @@ -1074,7 +1090,12 @@ njs_regexp_exec_result(njs_vm_t *vm, njs index = captures[1]; } - njs_set_number(®exp->last_index, index); + njs_set_number(&value, index); + ret = njs_value_property_set(vm, r, njs_value_arg(&njs_string_lindex), + &value); + if (njs_slow_path(ret != NJS_OK)) { + goto fail; + } } lhq.key_hash = NJS_INDEX_HASH; @@ -1270,7 +1291,6 @@ njs_regexp_prototype_symbol_replace(njs_ static const njs_value_t string_global = njs_string("global"); static const njs_value_t string_groups = njs_string("groups"); static const njs_value_t string_index = njs_string("index"); - static const njs_value_t string_lindex = njs_string("lastIndex"); rx = njs_argument(args, 0); @@ -1310,7 +1330,7 @@ njs_regexp_prototype_symbol_replace(njs_ if (global) { njs_set_number(&value, 0); - ret = njs_value_property_set(vm, rx, njs_value_arg(&string_lindex), + ret = njs_value_property_set(vm, rx, njs_value_arg(&njs_string_lindex), &value); if (njs_slow_path(ret != NJS_OK)) { return NJS_ERROR; @@ -1362,7 +1382,8 @@ njs_regexp_prototype_symbol_replace(njs_ continue; } - ret = njs_value_property(vm, rx, njs_value_arg(&string_lindex), &value); + ret = njs_value_property(vm, rx, njs_value_arg(&njs_string_lindex), + &value); if (njs_slow_path(ret == NJS_ERROR)) { goto exception; } @@ -1373,7 +1394,7 @@ njs_regexp_prototype_symbol_replace(njs_ } njs_set_number(&value, last_index + 1); - ret = njs_value_property_set(vm, rx, njs_value_arg(&string_lindex), + ret = njs_value_property_set(vm, rx, njs_value_arg(&njs_string_lindex), &value); if (njs_slow_path(ret != NJS_OK)) { goto exception; diff -r de64420d0f2b -r 45b9af3f911f src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Sat Jun 05 11:55:08 2021 +0000 +++ b/src/test/njs_unit_test.c Sat Jun 05 11:55:08 2021 +0000 @@ -10797,6 +10797,12 @@ static njs_unit_test_t njs_test[] = { njs_str("var re = /(?:ab|cd)\\d?/g; re.lastIndex=-1; re.test('@@'); re.lastIndex"), njs_str("0") }, + { njs_str("var r = /a/; var gets = 0;" + "var counter = { valueOf: function() { gets++; return 0; } };" + "r.lastIndex = counter;" + "njs.dump([r.exec('nbc'), r.lastIndex === counter, gets])"), + njs_str("[null,true,1]") }, + /* * It seems that "/????/ig" fails on early PCRE versions. * It fails at least in 8.1 and works at least in 8.31. From xeioex at nginx.com Tue Jun 8 17:59:58 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 08 Jun 2021 17:59:58 +0000 Subject: [njs] Fixed use-of-uninitialized-value while tracking rejected promises. Message-ID: details: https://hg.nginx.org/njs/rev/06204b343066 branches: changeset: 1653:06204b343066 user: Dmitry Volyntsev date: Tue Jun 08 12:43:13 2021 +0000 description: Fixed use-of-uninitialized-value while tracking rejected promises. Found by Found by MemorySanitizer. diffstat: src/njs_promise.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 45b9af3f911f -r 06204b343066 src/njs_promise.c --- a/src/njs_promise.c Sat Jun 05 11:55:08 2021 +0000 +++ b/src/njs_promise.c Tue Jun 08 12:43:13 2021 +0000 @@ -570,7 +570,7 @@ njs_promise_host_rejection_tracker(njs_v length = vm->promise_reason->length; for (i = 0; i < length; i++) { - if (memcmp(&value[i], &data->result, sizeof(njs_value_t)) == 0) { + if (njs_values_same(&value[i], &data->result)) { length--; if (i < length) { From xeioex at nginx.com Tue Jun 8 18:00:00 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 08 Jun 2021 18:00:00 +0000 Subject: [njs] Introduced RegExp.prototype.sticky support. Message-ID: details: https://hg.nginx.org/njs/rev/913a69870b49 branches: changeset: 1654:913a69870b49 user: Dmitry Volyntsev date: Sat Jun 05 11:55:08 2021 +0000 description: Introduced RegExp.prototype.sticky support. diffstat: src/njs_regexp.c | 60 +++++++++++++++++++++++++++++++++++++++++------ src/njs_regexp.h | 1 + src/njs_regexp_pattern.h | 1 + src/test/njs_unit_test.c | 33 +++++++++++++++++++++++--- 4 files changed, 83 insertions(+), 12 deletions(-) diffs (252 lines): diff -r 06204b343066 -r 913a69870b49 src/njs_regexp.c --- a/src/njs_regexp.c Tue Jun 08 12:43:13 2021 +0000 +++ b/src/njs_regexp.c Sat Jun 05 11:55:08 2021 +0000 @@ -93,6 +93,10 @@ njs_regexp_value_flags(njs_vm_t *vm, con flags |= NJS_REGEXP_MULTILINE; } + if (pattern->sticky) { + flags |= NJS_REGEXP_STICKY; + } + return flags; } @@ -332,6 +336,10 @@ njs_regexp_flags(u_char **start, u_char flag = NJS_REGEXP_MULTILINE; break; + case 'y': + flag = NJS_REGEXP_STICKY; + break; + default: if (*p >= 'a' && *p <= 'z') { goto invalid; @@ -429,6 +437,11 @@ njs_regexp_pattern_create(njs_vm_t *vm, options |= PCRE_MULTILINE; } + pattern->sticky = ((flags & NJS_REGEXP_STICKY) != 0); + if (pattern->sticky) { + options |= PCRE_ANCHORED; + } + *p++ = '\0'; ret = njs_regexp_pattern_compile(vm, &pattern->regex[0], @@ -644,11 +657,12 @@ njs_regexp_prototype_flags(njs_vm_t *vm, u_char *p; njs_int_t ret; njs_value_t *this, value; - u_char dst[3]; + u_char dst[4]; static const njs_value_t string_global = njs_string("global"); static const njs_value_t string_ignore_case = njs_string("ignoreCase"); static const njs_value_t string_multiline = njs_string("multiline"); + static const njs_value_t string_sticky = njs_string("sticky"); this = njs_argument(args, 0); if (njs_slow_path(!njs_is_object(this))) { @@ -688,6 +702,16 @@ njs_regexp_prototype_flags(njs_vm_t *vm, *p++ = 'm'; } + ret = njs_value_property(vm, this, njs_value_arg(&string_sticky), + &value); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + if (njs_bool(&value)) { + *p++ = 'y'; + } + return njs_string_new(vm, &vm->retval, dst, p - dst, p - dst); } @@ -728,8 +752,12 @@ njs_regexp_prototype_flag(njs_vm_t *vm, break; case NJS_REGEXP_MULTILINE: + yn = pattern->multiline; + break; + + case NJS_REGEXP_STICKY: default: - yn = pattern->multiline; + yn = pattern->sticky; break; } @@ -859,15 +887,19 @@ njs_regexp_to_string(njs_vm_t *vm, njs_v size = njs_strlen(source); length = njs_utf8_length(source, size); - length = (length >= 0) ? length: 0; + length = (length >= 0) ? (length + (pattern->sticky != 0)): 0; - p = njs_string_alloc(vm, retval, size, length); + p = njs_string_alloc(vm, retval, size + (pattern->sticky != 0), length); if (njs_slow_path(p == NULL)) { return NJS_ERROR; } p = njs_cpymem(p, source, size); + if (pattern->sticky) { + *p++ = 'y'; + } + return NJS_OK; } @@ -905,7 +937,7 @@ njs_regexp_prototype_test(njs_vm_t *vm, /** - * TODO: sticky, unicode flags. + * TODO: unicode flags. */ static njs_int_t njs_regexp_builtin_exec(njs_vm_t *vm, njs_value_t *r, njs_value_t *s, @@ -937,7 +969,7 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj return NJS_ERROR; } - if (!pattern->global) { + if (!pattern->global && !pattern->sticky) { last_index = 0; } @@ -995,7 +1027,7 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj not_found: - if (pattern->global) { + if (pattern->global || pattern->sticky) { njs_set_number(&value, 0); ret = njs_value_property_set(vm, r, njs_value_arg(&njs_string_lindex), &value); @@ -1082,7 +1114,7 @@ njs_regexp_exec_result(njs_vm_t *vm, njs njs_set_number(&prop->value, index); - if (pattern->global) { + if (pattern->global || pattern->sticky) { if (type == NJS_REGEXP_UTF8) { index = njs_string_index(string, captures[1]); @@ -1682,6 +1714,18 @@ static const njs_object_prop_t njs_rege { .type = NJS_PROPERTY, + .name = njs_string("sticky"), + .value = njs_value(NJS_INVALID, 1, NAN), + .getter = njs_native_function2(njs_regexp_prototype_flag, 0, + NJS_REGEXP_STICKY), + .setter = njs_value(NJS_UNDEFINED, 0, NAN), + .writable = NJS_ATTRIBUTE_UNSET, + .configurable = 1, + .enumerable = 0, + }, + + { + .type = NJS_PROPERTY, .name = njs_string("toString"), .value = njs_native_function(njs_regexp_prototype_to_string, 0), .writable = 1, diff -r 06204b343066 -r 913a69870b49 src/njs_regexp.h --- a/src/njs_regexp.h Tue Jun 08 12:43:13 2021 +0000 +++ b/src/njs_regexp.h Sat Jun 05 11:55:08 2021 +0000 @@ -14,6 +14,7 @@ typedef enum { NJS_REGEXP_GLOBAL = 1, NJS_REGEXP_IGNORE_CASE = 2, NJS_REGEXP_MULTILINE = 4, + NJS_REGEXP_STICKY = 8, } njs_regexp_flags_t; diff -r 06204b343066 -r 913a69870b49 src/njs_regexp_pattern.h --- a/src/njs_regexp_pattern.h Tue Jun 08 12:43:13 2021 +0000 +++ b/src/njs_regexp_pattern.h Sat Jun 05 11:55:08 2021 +0000 @@ -38,6 +38,7 @@ struct njs_regexp_pattern_s { uint8_t global; /* 1 bit */ uint8_t ignore_case; /* 1 bit */ uint8_t multiline; /* 1 bit */ + uint8_t sticky; /* 1 bit */ njs_regexp_group_t *groups; }; diff -r 06204b343066 -r 913a69870b49 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Tue Jun 08 12:43:13 2021 +0000 +++ b/src/test/njs_unit_test.c Sat Jun 05 11:55:08 2021 +0000 @@ -10631,6 +10631,12 @@ static njs_unit_test_t njs_test[] = { njs_str("/[A-Z/]/"), njs_str("/[A-Z/]/") }, + { njs_str("/a/gim"), + njs_str("/a/gim") }, + + { njs_str("/a/y"), + njs_str("/a/y") }, + { njs_str("/[A-Z\n]/"), njs_str("SyntaxError: Unterminated RegExp \"/[A-Z\" in 1") }, @@ -10720,6 +10726,9 @@ static njs_unit_test_t njs_test[] = { njs_str("/?/.test('\\u00CE\\u00B1'.toBytes())"), njs_str("true") }, + { njs_str("var r = /abc/y; r.test('abc'); r.lastIndex"), + njs_str("3") }, + { njs_str("/\\d/.exec('123')"), njs_str("1") }, @@ -10849,8 +10858,21 @@ static njs_unit_test_t njs_test[] = { njs_str("var r = (/^.+$/mg), s = 'ab\\nc'; [r.exec(s), r.exec(s)]"), njs_str("ab,c") }, - { njs_str("var r = (/^.+$/mg); [r.global, r.multiline, r.ignoreCase]"), - njs_str("true,true,false") }, + { njs_str("var r = (/^.+$/mg); [r.global, r.multiline, r.ignoreCase, r.sticky]"), + njs_str("true,true,false,false") }, + + { njs_str("['global', 'ignoreCase', 'multiline', 'sticky']" + ".map(v => Object.getOwnPropertyDescriptor(RegExp.prototype, v))" + ".every(desc => (typeof desc.get === 'function' && typeof desc.set === 'undefined'))"), + njs_str("true") }, + + { njs_str("var re = /./, re2 = /./y; re.lastIndex = 1; re2.lastIndex = 1;" + "[re.exec('abc')[0], re2.exec('abc')[0]]"), + njs_str("a,b") }, + + { njs_str("var re = /c/, re2 = /c/y;" + "njs.dump([re.exec('abc')[0], re2.exec('abc')])"), + njs_str("['c',null]") }, { njs_str("['global', 'ignoreCase', 'multiline']" ".map(v => Object.getOwnPropertyDescriptor(RegExp.prototype, v))" @@ -10869,6 +10891,9 @@ static njs_unit_test_t njs_test[] = { njs_str("var r = new RegExp('.', 'igm'); r"), njs_str("/./gim") }, + { njs_str("var r = new RegExp('.', 'y'); r"), + njs_str("/./y") }, + { njs_str("var r = new RegExp('abc'); r.test('00abc11')"), njs_str("true") }, @@ -17340,8 +17365,8 @@ static njs_unit_test_t njs_test[] = { njs_str("njs.dump({a:1, b:[1,,2,{c:new Boolean(1)}]})"), njs_str("{a:1,b:[1,,2,{c:[Boolean: true]}]}") }, - { njs_str("njs.dump([InternalError(),TypeError('msg'), new RegExp(), /^undef$/m, new Date(0)])"), - njs_str("[InternalError,TypeError: msg,/(?:)/,/^undef$/m,1970-01-01T00:00:00.000Z]") }, + { njs_str("njs.dump([InternalError(),TypeError('msg'), new RegExp(), /^undef$/my, new Date(0)])"), + njs_str("[InternalError,TypeError: msg,/(?:)/,/^undef$/my,1970-01-01T00:00:00.000Z]") }, { njs_str("njs.dump(Array.prototype.slice.call({'1':'b', length:2}))"), njs_str("[,'b']") }, From arut at nginx.com Wed Jun 9 12:10:13 2021 From: arut at nginx.com (Roman Arutyunyan) Date: Wed, 9 Jun 2021 15:10:13 +0300 Subject: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes In-Reply-To: <5da9c62fa61016600f2c.1622750089@vm-home1.archive.org> References: <5da9c62fa61016600f2c.1622750089@vm-home1.archive.org> Message-ID: <20210609121013.5tfxi2bao6rqkr7j@Romans-MacBook-Pro.local> Hello Tracey. Thanks for your patch, it looks very interesting. On Thu, Jun 03, 2021 at 07:54:49PM +0000, Tracey Jaquith wrote: > # HG changeset patch > # User Tracey Jaquith > # Date 1622678642 0 > # Thu Jun 03 00:04:02 2021 +0000 > # Node ID 5da9c62fa61016600f2c59982ae184e2811be427 > # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 > Add optional "&exact=1" CGI arg to show video between keyframes. I think this can be a directive (like mp4_buffer_size/mp4_max_buffer_size) rather than an argument. Is it possible than we need both exact and non-exact in the same location? It feels like we don't need to give clients control over this. The exact mode introduces some overhead and should only be used when needed. > archive.org has been using mod_h264_streaming with an "&exact=1" patch from me since 2013. > We just moved to nginx mp4 module and are using this patch. > The technique is to find the keyframe just before the desired "start" time, and send > that down the wire so video playback can start immediately. > Next calculate how many video samples are between the keyframe and desired "start" time > and update the STTS atom where those samples move the duration from (typically) 1001 to 1. > This way, initial unwanted video frames play at ~1/30,000s -- so visually the > video & audio start playing immediately. > > You can see an example before/after here (nginx binary built with mp4 module + patch): > > https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30 > https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30&exact=1 > > Tested on linux and macosx. > > I realize two var declarations should style-wise get moved "up" in the lower hooked in method. > However, to minimize the changeset/patch, I figured we should at least start here and see what you folks think. Syle-wise there are a few minor issues. We'll figure these out later. Below are my comments on ways to simplify the code. > (this is me: https://github.com/traceypooh ) > > diff -r 5f765427c17a -r 5da9c62fa610 src/http/modules/ngx_http_mp4_module.c > --- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300 > +++ b/src/http/modules/ngx_http_mp4_module.c Thu Jun 03 00:04:02 2021 +0000 > @@ -2045,6 +2045,109 @@ > u_char duration[4]; > } ngx_mp4_stts_entry_t; > > +typedef struct { > + uint32_t speedup_samples; > + ngx_uint_t speedup_seconds; > +} ngx_mp4_exact_t; > + > +static void > +exact_video_adjustment(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_mp4_exact_t *exact) > +{ > + // will parse STTS -- time-to-sample atom > + ngx_str_t value; > + ngx_buf_t *stts_data; > + ngx_buf_t *atom; > + ngx_mp4_stts_entry_t *stts_entry, *stts_end; > + uint32_t count, duration, j, n, sample_keyframe, sample_num; > + uint64_t sample_time, seconds, start_seconds_closest_keyframe; > + uint8_t is_keyframe; > + > + exact->speedup_samples = 0; > + exact->speedup_seconds = 0; > + > + // if '&exact=1' CGI arg isn't present, do nothing > + if (!(ngx_http_arg(mp4->request, (u_char *) "exact", 5, &value) == NGX_OK)) { > + return; > + } > + > + if (!trak->sync_samples_entries) { > + // Highly unlikely video STSS got parsed and _every_ sample is a keyframe. > + // However, if the case, we don't need to adjust the video at all. > + return; > + } > + > + // check HDLR atom to see if this trak is video or audio > + atom = trak->out[NGX_HTTP_MP4_HDLR_ATOM].buf; > + // 'vide' or 'soun' > + if (!(atom->pos[16] == 'v' && > + atom->pos[17] == 'i' && > + atom->pos[18] == 'd' && > + atom->pos[19] == 'e')) { > + return; // do nothing if not video > + } Do we really need to check this? If a track has an stss atom, this seems enough to do the work. > + stts_data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf; > + stts_entry = (ngx_mp4_stts_entry_t *) stts_data->pos; > + stts_end = (ngx_mp4_stts_entry_t *) stts_data->last; > + > + sample_num = 0; // they start at one > + sample_time = 0; > + start_seconds_closest_keyframe = 0; > + while (stts_entry < stts_end) { > + // STTS === time-to-sample atom > + // each entry is 4B and [sample count][sample duration] (since durations can vary) > + count = ngx_mp4_get_32value(stts_entry->count); > + duration = ngx_mp4_get_32value(stts_entry->duration); > + > + for (j = 0; j < count; j++) { > + sample_num++; > + > + // search STSS sync sample entries to see if this sample is a keyframe > + is_keyframe = (trak->sync_samples_entries ? 0 : 1); Considering the above, trak->sync_samples_entries is always non-zero here. > + for (n = 0; n < trak->sync_samples_entries; n++) { > + // each one of this these are a video sample number keyframe > + sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4)); > + if (sample_keyframe == sample_num) { > + is_keyframe = 1; > + break; > + } > + if (sample_keyframe > sample_num) { > + break; > + } > + } > + > + seconds = sample_time * 1000 / trak->timescale; > + sample_time += duration; > + > + if (seconds > mp4->start) { > + goto found; > + } > + > + if (is_keyframe) { > + start_seconds_closest_keyframe = seconds; > + exact->speedup_samples = 0; > + } else { > + exact->speedup_samples++; > + } > + } > + > + stts_entry++; > + } > + > + found: > + > + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, > + "exact_video_adjustment() new keyframe start: %d, speedup first %d samples", > + start_seconds_closest_keyframe, > + exact->speedup_samples); > + > + // NOTE: begin 1 start position before keyframe to ensure first video frame emitted is always > + // a keyframe > + exact->speedup_seconds = mp4->start - start_seconds_closest_keyframe > + - (start_seconds_closest_keyframe ? 1 : 0); > +} > + > > static ngx_int_t > ngx_http_mp4_read_stts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) > @@ -2164,10 +2267,14 @@ > ngx_buf_t *data; > ngx_uint_t start_sample, entries, start_sec; > ngx_mp4_stts_entry_t *entry, *end; > + ngx_mp4_exact_t exact; > > if (start) { > start_sec = mp4->start; > > + exact_video_adjustment(mp4, trak, &exact); > + start_sec -= exact.speedup_seconds; Here you find the closest key frame from the past and adjust start_sec to match it. But what if we don't do this here, but continue with the original start_sec. When we find the right (probably non-key) frame, we'll have its number in start_sample. All we'll need to do at that point is to search for the closest key frame to that sample number - just a simple loop. When we find the number of samples into the past until the most recent key frame, we'll just add one entry to the stts array with (this number, 1) and then decrease start_sample accordingly. Sounds simpler. > + > ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, > "mp4 stts crop start_time:%ui", start_sec); > > @@ -2230,6 +2337,42 @@ > > if (start) { > ngx_mp4_set_32value(entry->count, count - rest); > + > + if (exact.speedup_samples) { > + // We're going to prepend an entry with duration=1 for the frames we want to "not see". > + // MOST of the time, we're taking a single element entry array and making it two. > + uint32_t current_count = ngx_mp4_get_32value(entry->count); > + ngx_mp4_stts_entry_t* entries_array = ngx_palloc(mp4->request->pool, > + (1 + entries) * sizeof(ngx_mp4_stts_entry_t)); > + if (entries_array == NULL) { > + return NGX_ERROR; > + } I believe there are situations when there's an entry from the past in the old array, that can be reused. > + ngx_copy(&(entries_array[1]), entry, entries * sizeof(ngx_mp4_stts_entry_t)); > + > + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, > + "exact split in 2 video STTS entry from count:%d", current_count); > + > + if (current_count <= exact.speedup_samples) > + return NGX_ERROR; > + > + entry = &(entries_array[1]); > + ngx_mp4_set_32value(entry->count, current_count - exact.speedup_samples); > + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, > + "exact split new[1]: count:%d duration:%d", > + ngx_mp4_get_32value(entry->count), > + ngx_mp4_get_32value(entry->duration)); > + entry--; > + ngx_mp4_set_32value(entry->count, exact.speedup_samples); > + ngx_mp4_set_32value(entry->duration, 1); > + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, > + "exact split new[0]: count:%d duration:1", > + ngx_mp4_get_32value(entry->count)); > + > + data->last = (u_char *) (entry + 2); > + > + entries++; > + } > + > data->pos = (u_char *) entry; > trak->time_to_sample_entries = entries; > trak->start_sample = start_sample; > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > http://mailman.nginx.org/mailman/listinfo/nginx-devel -- Roman Arutyunyan From xeioex at nginx.com Wed Jun 9 17:54:13 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Wed, 09 Jun 2021 17:54:13 +0000 Subject: [njs] Added benchmarks for String.prototype.split(). Message-ID: details: https://hg.nginx.org/njs/rev/5516f717a8c9 branches: changeset: 1655:5516f717a8c9 user: Dmitry Volyntsev date: Tue Jun 08 18:01:25 2021 +0000 description: Added benchmarks for String.prototype.split(). diffstat: src/test/njs_benchmark.c | 15 +++++++++++++++ 1 files changed, 15 insertions(+), 0 deletions(-) diffs (25 lines): diff -r 913a69870b49 -r 5516f717a8c9 src/test/njs_benchmark.c --- a/src/test/njs_benchmark.c Sat Jun 05 11:55:08 2021 +0000 +++ b/src/test/njs_benchmark.c Tue Jun 08 18:01:25 2021 +0000 @@ -316,6 +316,21 @@ static njs_benchmark_test_t njs_test[] njs_str("undefined"), 1 }, + { "regexp split", + njs_str("'a a'.split(/ /).length"), + njs_str("2"), + 100 }, + + { "regexp 10K split", + njs_str("'a '.repeat(10000).split(/ /).length"), + njs_str("10001"), + 1 }, + + { "simple 100K split", + njs_str("'a '.repeat(100000).split(' ').length"), + njs_str("100001"), + 1 }, + { "external property ($shared.uri)", njs_str("$shared.uri"), njs_str("shared"), From xeioex at nginx.com Wed Jun 9 17:54:15 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Wed, 09 Jun 2021 17:54:15 +0000 Subject: [njs] Fixed String.prototype.split() according to the specification. Message-ID: details: https://hg.nginx.org/njs/rev/f2d02c3b5f8a branches: changeset: 1656:f2d02c3b5f8a user: Dmitry Volyntsev date: Wed Jun 09 17:14:10 2021 +0000 description: Fixed String.prototype.split() according to the specification. This closes #359 issue on GitHub. diffstat: src/njs_regexp.c | 252 ++++++++++++++++++++++++++++++++++++++++++ src/njs_string.c | 278 ++++++++++++++++++++-------------------------- src/njs_string.h | 2 + src/test/njs_benchmark.c | 6 +- src/test/njs_unit_test.c | 38 ++++++- 5 files changed, 418 insertions(+), 158 deletions(-) diffs (697 lines): diff -r 5516f717a8c9 -r f2d02c3b5f8a src/njs_regexp.c --- a/src/njs_regexp.c Tue Jun 08 18:01:25 2021 +0000 +++ b/src/njs_regexp.c Wed Jun 09 17:14:10 2021 +0000 @@ -1612,6 +1612,250 @@ exception: } +static njs_int_t +njs_regexp_prototype_symbol_split(njs_vm_t *vm, njs_value_t *args, + njs_uint_t nargs, njs_index_t unused) +{ + u_char *dst; + int64_t e, i, p, q, ncaptures, length; + uint32_t limit; + njs_int_t ret; + njs_bool_t sticky; + njs_utf8_t utf8; + njs_array_t *array; + njs_value_t *rx, *string, *value; + njs_value_t r, z, this, s_lvalue, retval, setval, constructor; + njs_object_t *object; + const u_char *start, *end; + njs_string_prop_t s; + njs_value_t arguments[2]; + + static const njs_value_t string_lindex = njs_string("lastIndex"); + static const njs_value_t string_flags = njs_string("flags"); + + rx = njs_argument(args, 0); + + if (njs_slow_path(!njs_is_object(rx))) { + njs_type_error(vm, "\"this\" is not object"); + return NJS_ERROR; + } + + string = njs_lvalue_arg(&s_lvalue, args, nargs, 1); + + ret = njs_value_to_string(vm, string, string); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + njs_set_function(&constructor, &vm->constructors[NJS_OBJ_TYPE_REGEXP]); + + ret = njs_value_species_constructor(vm, rx, &constructor, &constructor); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + ret = njs_value_property(vm, rx, njs_value_arg(&string_flags), &retval); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + ret = njs_value_to_string(vm, &retval, &retval); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + (void) njs_string_prop(&s, &retval); + + sticky = memchr(s.start, 'y', s.size) != NULL; + + object = njs_function_new_object(vm, &constructor); + if (njs_slow_path(object == NULL)) { + return NJS_ERROR; + } + + njs_set_object(&this, object); + + arguments[0] = *rx; + + if (!sticky) { + length = njs_is_byte_string(&s) ? 0 : s.length + 1; + + dst = njs_string_alloc(vm, &arguments[1], s.size + 1, length); + if (njs_slow_path(dst == NULL)) { + return NJS_ERROR; + } + + dst = njs_cpymem(dst, s.start, s.size); + *dst++ = 'y'; + + } else { + arguments[1] = retval; + } + + ret = njs_function_call2(vm, njs_function(&constructor), &this, + njs_value_arg(&arguments), 2, &r, 1); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + rx = &r; + + array = njs_array_alloc(vm, 0, 0, NJS_ARRAY_SPARE); + if (njs_slow_path(array == NULL)) { + return NJS_ERROR; + } + + value = njs_arg(args, nargs, 2); + limit = UINT32_MAX; + + if (njs_is_defined(value)) { + ret = njs_value_to_uint32(vm, value, &limit); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + } + + if (njs_slow_path(limit == 0)) { + goto done; + } + + length = njs_string_prop(&s, string); + + if (njs_slow_path(s.size == 0)) { + ret = njs_regexp_exec(vm, rx, string, &z); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + if (!njs_is_null(&z)) { + goto done; + } + + goto single; + } + + utf8 = NJS_STRING_BYTE; + + if (s.length != 0 && s.length != s.size) { + utf8 = NJS_STRING_UTF8; + } + + p = 0; + q = 0; + + while (q < length) { + njs_set_number(&setval, q); + ret = njs_value_property_set(vm, rx, njs_value_arg(&string_lindex), + &setval); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + ret = njs_regexp_exec(vm, rx, string, &z); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + if (njs_is_null(&z)) { + q = q + 1; + continue; + } + + ret = njs_value_property(vm, rx, njs_value_arg(&string_lindex), + &retval); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + ret = njs_value_to_length(vm, &retval, &e); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + e = njs_min(e, length); + + if (e == p) { + q = q + 1; + continue; + } + + if (utf8 == NJS_STRING_UTF8) { + start = njs_string_offset(s.start, s.start + s.size, p); + end = njs_string_offset(s.start, s.start + s.size, q); + + } else { + start = &s.start[p]; + end = &s.start[q]; + } + + ret = njs_string_split_part_add(vm, array, utf8, start, end - start); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + if (array->length == limit) { + goto done; + } + + p = e; + + ret = njs_object_length(vm, &z, &ncaptures); + if (njs_slow_path(ret != NJS_OK)) { + return NJS_ERROR; + } + + ncaptures = njs_max(ncaptures - 1, 0); + + for (i = 1; i <= ncaptures; i++) { + value = njs_array_push(vm, array); + if (njs_slow_path(value == NULL)) { + return NJS_ERROR; + } + + ret = njs_value_property_i64(vm, &z, i, value); + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; + } + + if (array->length == limit) { + goto done; + } + } + + q = p; + } + + end = &s.start[s.size]; + + if (utf8 == NJS_STRING_UTF8) { + start = njs_string_offset(s.start, s.start + s.size, p); + + } else { + start = &s.start[p]; + } + + ret = njs_string_split_part_add(vm, array, utf8, start, end - start); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + goto done; + +single: + + value = njs_array_push(vm, array); + if (njs_slow_path(value == NULL)) { + return NJS_ERROR; + } + + *value = *string; + +done: + + njs_set_array(&vm->retval, array); + + return NJS_OK; +} static const njs_object_prop_t njs_regexp_constructor_properties[] = @@ -1755,6 +1999,14 @@ static const njs_object_prop_t njs_rege .writable = 1, .configurable = 1, }, + + { + .type = NJS_PROPERTY, + .name = njs_wellknown_symbol(NJS_SYMBOL_SPLIT), + .value = njs_native_function(njs_regexp_prototype_symbol_split, 2), + .writable = 1, + .configurable = 1, + }, }; diff -r 5516f717a8c9 -r f2d02c3b5f8a src/njs_string.c --- a/src/njs_string.c Tue Jun 08 18:01:25 2021 +0000 +++ b/src/njs_string.c Wed Jun 09 17:14:10 2021 +0000 @@ -72,8 +72,6 @@ static njs_int_t njs_string_bytes_from_s const njs_value_t *string, const njs_value_t *encoding); static njs_int_t njs_string_match_multiple(njs_vm_t *vm, njs_value_t *args, njs_regexp_pattern_t *pattern); -static njs_int_t njs_string_split_part_add(njs_vm_t *vm, njs_array_t *array, - njs_utf8_t utf8, const u_char *start, size_t size); #define njs_base64_encoded_length(len) (((len + 2) / 3) * 4) @@ -3338,19 +3336,49 @@ static njs_int_t njs_string_prototype_split(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused) { - int *captures; - size_t size; - uint32_t limit; - njs_int_t ret; - njs_utf8_t utf8; - njs_value_t *value; - njs_array_t *array; - const u_char *p, *start, *next, *last, *end; - njs_regexp_utf8_t type; - njs_string_prop_t string, split; - njs_regexp_pattern_t *pattern; - - ret = njs_string_object_validate(vm, njs_arg(args, nargs, 0)); + size_t size; + uint32_t limit; + njs_int_t ret; + njs_utf8_t utf8; + njs_bool_t undefined; + njs_value_t *this, *separator, *value; + njs_value_t separator_lvalue, limit_lvalue, splitter; + njs_array_t *array; + const u_char *p, *start, *next, *last, *end; + njs_string_prop_t string, split; + njs_value_t arguments[3]; + + static const njs_value_t split_key = + njs_wellknown_symbol(NJS_SYMBOL_SPLIT); + + this = njs_argument(args, 0); + + if (njs_slow_path(njs_is_null_or_undefined(this))) { + njs_type_error(vm, "cannot convert \"%s\"to object", + njs_type_string(this->type)); + return NJS_ERROR; + } + + separator = njs_lvalue_arg(&separator_lvalue, args, nargs, 1); + value = njs_lvalue_arg(&limit_lvalue, args, nargs, 2); + + if (!njs_is_null_or_undefined(separator)) { + ret = njs_value_method(vm, separator, njs_value_arg(&split_key), + &splitter); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + if (njs_is_defined(&splitter)) { + arguments[0] = *this; + arguments[1] = *value; + + return njs_function_call(vm, njs_function(&splitter), separator, + arguments, 2, &vm->retval); + } + } + + ret = njs_value_to_string(vm, this, this); if (njs_slow_path(ret != NJS_OK)) { return ret; } @@ -3360,159 +3388,99 @@ njs_string_prototype_split(njs_vm_t *vm, return NJS_ERROR; } - if (nargs > 1) { - - if (nargs > 2) { - value = njs_argument(args, 2); - - if (njs_slow_path(!njs_is_number(value))) { - ret = njs_value_to_uint32(vm, value, &limit); - if (njs_slow_path(ret != NJS_OK)) { - return ret; - } - - } else { - limit = njs_number_to_uint32(njs_number(value)); - } - - if (limit == 0) { - goto done; - } - - } else { - limit = (uint32_t) -1; + limit = UINT32_MAX; + + if (njs_is_defined(value)) { + ret = njs_value_to_uint32(vm, value, &limit); + if (njs_slow_path(ret != NJS_OK)) { + return ret; } - - (void) njs_string_prop(&string, &args[0]); - - if (string.size == 0) { + } + + undefined = njs_is_undefined(separator); + + ret = njs_value_to_string(vm, separator, separator); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + if (njs_slow_path(limit == 0)) { + goto done; + } + + if (njs_slow_path(undefined)) { + goto single; + } + + (void) njs_string_prop(&string, this); + (void) njs_string_prop(&split, separator); + + if (njs_slow_path(string.size == 0)) { + if (split.size != 0) { goto single; } - utf8 = NJS_STRING_BYTE; - type = NJS_REGEXP_BYTE; - - if (string.length != 0) { - utf8 = NJS_STRING_ASCII; - type = NJS_REGEXP_UTF8; - - if (string.length != string.size) { - utf8 = NJS_STRING_UTF8; + goto done; + } + + utf8 = NJS_STRING_BYTE; + + if (string.length != 0) { + utf8 = NJS_STRING_ASCII; + + if (string.length != string.size) { + utf8 = NJS_STRING_UTF8; + } + } + + start = string.start; + end = string.start + string.size; + last = end - split.size; + + do { + + for (p = start; p <= last; p++) { + if (memcmp(p, split.start, split.size) == 0) { + goto found; } } - switch (args[1].type) { - - case NJS_REGEXP: - pattern = njs_regexp_pattern(&args[1]); - - if (!njs_regex_is_valid(&pattern->regex[type])) { - goto single; - } - - start = string.start; - end = string.start + string.size; - - do { - ret = njs_regexp_match(vm, &pattern->regex[type], start, 0, - end - start, vm->single_match_data); - if (ret >= 0) { - captures = njs_regex_captures(vm->single_match_data); - - p = start + captures[0]; - next = start + captures[1]; - - } else if (ret == NJS_REGEX_NOMATCH) { - p = (u_char *) end; - next = (u_char *) end + 1; - - } else { - return NJS_ERROR; - } - - /* Empty split regexp. */ - if (p == next) { - p = (utf8 != NJS_STRING_BYTE) ? njs_utf8_next(p, end) - : p + 1; - next = p; - } - - size = p - start; - - ret = njs_string_split_part_add(vm, array, utf8, start, size); - if (njs_slow_path(ret != NJS_OK)) { - return ret; - } - - start = next; - limit--; - - } while (limit != 0 && p < end); - - goto done; - - case NJS_UNDEFINED: - break; - - default: - if (njs_slow_path(!njs_is_string(&args[1]))) { - ret = njs_value_to_string(vm, &args[1], &args[1]); - if (njs_slow_path(ret != NJS_OK)) { - return ret; - } - } - - (void) njs_string_prop(&split, &args[1]); - - if (string.size < split.size) { - goto single; - } - - start = string.start; - end = string.start + string.size; - last = end - split.size; - - do { - for (p = start; p <= last; p++) { - if (memcmp(p, split.start, split.size) == 0) { - goto found; - } - } - - p = end; + p = end; found: - next = p + split.size; - - /* Empty split string. */ - if (p == next) { - p = (utf8 != NJS_STRING_BYTE) ? njs_utf8_next(p, end) - : p + 1; - next = p; - } - - size = p - start; - - ret = njs_string_split_part_add(vm, array, utf8, start, size); - if (njs_slow_path(ret != NJS_OK)) { - return ret; - } - - start = next; - limit--; - - } while (limit != 0 && p < end); - - goto done; + next = p + split.size; + + /* Empty split string. */ + + if (p == next) { + p = (utf8 != NJS_STRING_BYTE) ? njs_utf8_next(p, end) + : p + 1; + next = p; } - } + + size = p - start; + + ret = njs_string_split_part_add(vm, array, utf8, start, size); + if (njs_slow_path(ret != NJS_OK)) { + return ret; + } + + start = next; + limit--; + + } while (limit != 0 && p < end); + + goto done; single: - /* GC: retain. */ - array->start[0] = args[0]; - array->length = 1; + value = njs_array_push(vm, array); + if (njs_slow_path(value == NULL)) { + return NJS_ERROR; + } + + *value = *this; done: @@ -3522,7 +3490,7 @@ done: } -static njs_int_t +njs_int_t njs_string_split_part_add(njs_vm_t *vm, njs_array_t *array, njs_utf8_t utf8, const u_char *start, size_t size) { diff -r 5516f717a8c9 -r f2d02c3b5f8a src/njs_string.h --- a/src/njs_string.h Tue Jun 08 18:01:25 2021 +0000 +++ b/src/njs_string.h Wed Jun 09 17:14:10 2021 +0000 @@ -239,6 +239,8 @@ njs_int_t njs_string_decode_uri(njs_vm_t njs_int_t njs_string_prototype_concat(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused); +njs_int_t njs_string_split_part_add(njs_vm_t *vm, njs_array_t *array, + njs_utf8_t utf8, const u_char *start, size_t size); njs_int_t njs_string_get_substitution(njs_vm_t *vm, njs_value_t *matched, njs_value_t *string, int64_t pos, njs_value_t *captures, int64_t ncaptures, njs_value_t *groups, njs_value_t *replacement, njs_value_t *retval); diff -r 5516f717a8c9 -r f2d02c3b5f8a src/test/njs_benchmark.c --- a/src/test/njs_benchmark.c Tue Jun 08 18:01:25 2021 +0000 +++ b/src/test/njs_benchmark.c Wed Jun 09 17:14:10 2021 +0000 @@ -317,8 +317,10 @@ static njs_benchmark_test_t njs_test[] 1 }, { "regexp split", - njs_str("'a a'.split(/ /).length"), - njs_str("2"), + njs_str("var s = Array(26).fill(0).map((v,i)=> {" + " var u = String.fromCodePoint(65+i), l = u.toLowerCase(); return u+l+l;}).join('');" + "s.split(/(?=[A-Z])/).length"), + njs_str("26"), 100 }, { "regexp 10K split", diff -r 5516f717a8c9 -r f2d02c3b5f8a src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Tue Jun 08 18:01:25 2021 +0000 +++ b/src/test/njs_unit_test.c Wed Jun 09 17:14:10 2021 +0000 @@ -8635,6 +8635,9 @@ static njs_unit_test_t njs_test[] = { njs_str("/[\\"), njs_str("SyntaxError: Unterminated RegExp \"/[\\\" in 1") }, + { njs_str("/\\s*;\\s*/"), + njs_str("/\\s*;\\s*/") }, + { njs_str("RegExp(']')"), njs_str("/\\]/") }, @@ -8802,7 +8805,7 @@ static njs_unit_test_t njs_test[] = njs_str("abc") }, { njs_str("''.split('').length"), - njs_str("1") }, + njs_str("0") }, { njs_str("'abc'.split('')"), njs_str("a,b,c") }, @@ -8858,9 +8861,40 @@ static njs_unit_test_t njs_test[] = { njs_str("'abc'.split(/abc/)"), njs_str(",") }, + { njs_str("'AbcDefGhi'.split(/([A-Z][a-z]+)/)"), + njs_str(",Abc,,Def,,Ghi,") }, + + { njs_str("'myCamelCaseString'.split(/(?=[A-Z])/)"), + njs_str("my,Camel,Case,String") }, + + { njs_str("'??????????????????'.split(/(?=[?-?])/)"), + njs_str("???,?????????,??????") }, + + { njs_str("'Harry Trump ;Fred Barney; Helen Rigby ; Bill Abel ;Chris Hand '.split( /\\s*(?:;|$)\\s*/)"), + njs_str("Harry Trump,Fred Barney,Helen Rigby,Bill Abel,Chris Hand,") }, + + { njs_str("'????? ????? ;???? ?????; ????? ????? ; ???? ?????'.split(/\\s*;\\s*/)"), + njs_str("????? ?????,???? ?????,????? ?????,???? ?????") }, + + { njs_str("'Hello 1 world. Sentence number 2.'.split(/(\\d)/)"), + njs_str("Hello ,1, world. Sentence number ,2,.") }, + + { njs_str("'?????? 1 ???. ??????????? ????? 2.'.split(/(\\d)/)"), + njs_str("?????? ,1, ???. ??????????? ????? ,2,.") }, + { njs_str("'0123456789'.split('').reverse().join('')"), njs_str("9876543210") }, + { njs_str("/-/[Symbol.split]('a-b-c')"), + njs_str("a,b,c") }, + + { njs_str("var O = RegExp.prototype[Symbol.split];" + "RegExp.prototype[Symbol.split] = function (s, limit) { " + " return O.call(this, s, limit).map(v => `@${v}#`); " + "};" + "'2016-01-02'.split(/-/)"), + njs_str("@2016#, at 01#, at 02#") }, + { njs_str("'abc'.repeat(3)"), njs_str("abcabcabc") }, @@ -17006,11 +17040,13 @@ static njs_unit_test_t njs_test[] = { njs_str("var a = [1]; a[2] = 'x'; JSON.stringify(a)"), njs_str("[1,null,\"x\"]") }, +#if (!NJS_HAVE_MEMORY_SANITIZER) /* very long test under MSAN */ { njs_str(njs_declare_sparse_array("a", 32769) "a[32] = 'a'; a[64] = 'b';" "var s = JSON.stringify(a); " "[s.length,s.substring(162,163),s.match(/null/g).length]"), njs_str("163844,a,32767") }, +#endif { njs_str(njs_declare_sparse_array("a", 8) "a[2] = 'a'; a[4] = 'b'; a.length = 3;" From nginx-devel at cybojanek.net Thu Jun 10 15:07:39 2021 From: nginx-devel at cybojanek.net (Jan Kasiak) Date: Thu, 10 Jun 2021 11:07:39 -0400 Subject: Format string bug Message-ID: <73876b44-eda3-4ed7-8b1d-3e43b5d9ae7e@www.fastmail.com> I think there is a bug in one of the format strings. diff --git a/src/os/win32/ngx_win32_init.c b/src/os/win32/ngx_win32_init.c index 3249fb29..11941f62 100644 --- a/src/os/win32/ngx_win32_init.c +++ b/src/os/win32/ngx_win32_init.c @@ -305,7 +305,7 @@ ngx_os_status(ngx_log_t *log) /* Win9x build */ ngx_log_error(NGX_LOG_INFO, log, 0, - "OS: %u build:%ud.%ud.%ud, \"%s\"", + "OS: %ud build:%ud.%ud.%ud, \"%s\"", ngx_win32_version, osvi.dwBuildNumber >> 24, (osvi.dwBuildNumber >> 16) & 0xff, From mdounin at mdounin.ru Thu Jun 10 16:51:32 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 10 Jun 2021 19:51:32 +0300 Subject: Format string bug In-Reply-To: <73876b44-eda3-4ed7-8b1d-3e43b5d9ae7e@www.fastmail.com> References: <73876b44-eda3-4ed7-8b1d-3e43b5d9ae7e@www.fastmail.com> Message-ID: Hello! On Thu, Jun 10, 2021 at 11:07:39AM -0400, Jan Kasiak wrote: > I think there is a bug in one of the format strings. > > diff --git a/src/os/win32/ngx_win32_init.c b/src/os/win32/ngx_win32_init.c > index 3249fb29..11941f62 100644 > --- a/src/os/win32/ngx_win32_init.c > +++ b/src/os/win32/ngx_win32_init.c > @@ -305,7 +305,7 @@ ngx_os_status(ngx_log_t *log) > /* Win9x build */ > > ngx_log_error(NGX_LOG_INFO, log, 0, > - "OS: %u build:%ud.%ud.%ud, \"%s\"", > + "OS: %ud build:%ud.%ud.%ud, \"%s\"", > ngx_win32_version, > osvi.dwBuildNumber >> 24, > (osvi.dwBuildNumber >> 16) & 0xff, Yes, thanks. Further, the "%ud" used elsewhere for ngx_win32_version is also incorrect, given the ngx_uint_t type: it should be %ui instead. The following patch fixes both issues: # HG changeset patch # User Maxim Dounin # Date 1623342077 -10800 # Thu Jun 10 19:21:17 2021 +0300 # Node ID f7120b838d9c7647a711e53166a6f0f2522b35da # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 Fixed format strings for ngx_win32_version. diff --git a/src/os/win32/ngx_win32_init.c b/src/os/win32/ngx_win32_init.c --- a/src/os/win32/ngx_win32_init.c +++ b/src/os/win32/ngx_win32_init.c @@ -295,7 +295,7 @@ ngx_os_status(ngx_log_t *log) osviex_stub = (ngx_osviex_stub_t *) &osvi.wServicePackMinor; ngx_log_error(NGX_LOG_INFO, log, 0, - "OS: %ud build:%ud, \"%s\", suite:%Xd, type:%ud", + "OS: %ui build:%ud, \"%s\", suite:%Xd, type:%ud", ngx_win32_version, osvi.dwBuildNumber, osvi.szCSDVersion, osviex_stub->wSuiteMask, osviex_stub->wProductType); @@ -305,7 +305,7 @@ ngx_os_status(ngx_log_t *log) /* Win9x build */ ngx_log_error(NGX_LOG_INFO, log, 0, - "OS: %u build:%ud.%ud.%ud, \"%s\"", + "OS: %ui build:%ud.%ud.%ud, \"%s\"", ngx_win32_version, osvi.dwBuildNumber >> 24, (osvi.dwBuildNumber >> 16) & 0xff, @@ -321,7 +321,7 @@ ngx_os_status(ngx_log_t *log) * and we do not support VER_PLATFORM_WIN32s at all */ - ngx_log_error(NGX_LOG_INFO, log, 0, "OS: %ud build:%ud, \"%s\"", + ngx_log_error(NGX_LOG_INFO, log, 0, "OS: %ui build:%ud, \"%s\"", ngx_win32_version, osvi.dwBuildNumber, osvi.szCSDVersion); } -- Maxim Dounin http://mdounin.ru/ From xeioex at nginx.com Fri Jun 11 15:21:32 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 11 Jun 2021 15:21:32 +0000 Subject: [njs] Fixed dumping of objects with circular references. Message-ID: details: https://hg.nginx.org/njs/rev/1c066436ec55 branches: changeset: 1657:1c066436ec55 user: Dmitry Volyntsev date: Fri Jun 11 15:15:48 2021 +0000 description: Fixed dumping of objects with circular references. This closes #395 issue on Github. diffstat: src/njs_json.c | 69 ++++++----------------------------------------- src/test/njs_unit_test.c | 9 ++++++ 2 files changed, 19 insertions(+), 59 deletions(-) diffs (150 lines): diff -r f2d02c3b5f8a -r 1c066436ec55 src/njs_json.c --- a/src/njs_json.c Wed Jun 09 17:14:10 2021 +0000 +++ b/src/njs_json.c Fri Jun 11 15:15:48 2021 +0000 @@ -2036,40 +2036,15 @@ njs_dump_is_recursive(const njs_value_t njs_inline njs_int_t -njs_dump_visit(njs_arr_t *list, const njs_value_t *value) +njs_dump_visited(njs_vm_t *vm, njs_json_stringify_t *stringify, + const njs_value_t *value) { - njs_object_t **p; - - if (njs_is_object(value)) { - p = njs_arr_add(list); - if (njs_slow_path(p == NULL)) { - return NJS_ERROR; - } - - *p = njs_object(value); - } - - return NJS_OK; -} - - -njs_inline njs_int_t -njs_dump_visited(njs_arr_t *list, const njs_value_t *value) -{ - njs_uint_t items, n; - njs_object_t **start, *obj; - - if (!njs_is_object(value)) { - /* External. */ - return 0; - } - - start = list->start; - items = list->items; - obj = njs_object(value); - - for (n = 0; n < items; n++) { - if (start[n] == obj) { + njs_int_t depth; + + depth = stringify->depth - 1; + + for (; depth >= 0; depth--) { + if (njs_values_same(&stringify->states[depth].value, value)) { return 1; } } @@ -2125,9 +2100,7 @@ njs_vm_value_dump(njs_vm_t *vm, njs_str_ njs_int_t ret; njs_chb_t chain; njs_str_t str; - njs_arr_t visited; njs_value_t *key, *val, tag; - njs_object_t **start; njs_json_state_t *state; njs_string_prop_t string; njs_object_prop_t *prop; @@ -2147,9 +2120,6 @@ njs_vm_value_dump(njs_vm_t *vm, njs_str_ goto memory_error; } - visited.separate = 0; - visited.pointer = 0; - goto done; } @@ -2166,13 +2136,6 @@ njs_vm_value_dump(njs_vm_t *vm, njs_str_ goto memory_error; } - start = njs_arr_init(vm->mem_pool, &visited, NULL, 8, sizeof(void *)); - if (njs_slow_path(start == NULL)) { - goto memory_error; - } - - (void) njs_dump_visit(&visited, value); - for ( ;; ) { switch (state->type) { case NJS_JSON_OBJECT: @@ -2276,16 +2239,11 @@ njs_vm_value_dump(njs_vm_t *vm, njs_str_ } if (njs_dump_is_recursive(val)) { - if (njs_slow_path(njs_dump_visited(&visited, val))) { + if (njs_slow_path(njs_dump_visited(vm, stringify, val))) { njs_chb_append_literal(&chain, "[Circular]"); break; } - ret = njs_dump_visit(&visited, val); - if (njs_slow_path(ret != NJS_OK)) { - goto memory_error; - } - state = njs_json_push_stringify_state(vm, stringify, val); if (njs_slow_path(state == NULL)) { goto exception; @@ -2342,16 +2300,11 @@ njs_vm_value_dump(njs_vm_t *vm, njs_str_ val = &njs_array_start(&state->value)[state->index++]; if (njs_dump_is_recursive(val)) { - if (njs_slow_path(njs_dump_visited(&visited, val))) { + if (njs_slow_path(njs_dump_visited(vm, stringify, val))) { njs_chb_append_literal(&chain, "[Circular]"); break; } - ret = njs_dump_visit(&visited, val); - if (njs_slow_path(ret != NJS_OK)) { - goto memory_error; - } - state = njs_json_push_stringify_state(vm, stringify, val); if (njs_slow_path(state == NULL)) { goto exception; @@ -2377,8 +2330,6 @@ njs_vm_value_dump(njs_vm_t *vm, njs_str_ done: - njs_arr_destroy(&visited); - ret = njs_chb_join(&chain, &str); if (njs_slow_path(ret != NJS_OK)) { goto memory_error; diff -r f2d02c3b5f8a -r 1c066436ec55 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Wed Jun 09 17:14:10 2021 +0000 +++ b/src/test/njs_unit_test.c Fri Jun 11 15:15:48 2021 +0000 @@ -17422,6 +17422,15 @@ static njs_unit_test_t njs_test[] = { njs_str("var a = []; a[0] = a; njs.dump(a)"), njs_str("[[Circular]]") }, + { njs_str("var a = []; njs.dump([a,a])"), + njs_str("[[],[]]") }, + + { njs_str("var O = {}; O.x = O; njs.dump(O)"), + njs_str("{x:[Circular]}") }, + + { njs_str("var O = {}; njs.dump({x:O, y:O})"), + njs_str("{x:{},y:{}}") }, + { njs_str("var a = [], b = [a]; a[0] = b; njs.dump(a)"), njs_str("[[[Circular]]]") }, From xeioex at nginx.com Fri Jun 11 18:28:40 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 11 Jun 2021 18:28:40 +0000 Subject: [njs] Fixed RegExp.prototype.[@@split]() with UTF8 strings. Message-ID: details: https://hg.nginx.org/njs/rev/a0f5719c8d9a branches: changeset: 1658:a0f5719c8d9a user: Dmitry Volyntsev date: Fri Jun 11 15:47:37 2021 +0000 description: Fixed RegExp.prototype.[@@split]() with UTF8 strings. Found by OSS-Fuzz. diffstat: src/njs_regexp.c | 3 ++- src/test/njs_unit_test.c | 3 +++ 2 files changed, 5 insertions(+), 1 deletions(-) diffs (26 lines): diff -r 1c066436ec55 -r a0f5719c8d9a src/njs_regexp.c --- a/src/njs_regexp.c Fri Jun 11 15:15:48 2021 +0000 +++ b/src/njs_regexp.c Fri Jun 11 15:47:37 2021 +0000 @@ -1828,7 +1828,8 @@ njs_regexp_prototype_symbol_split(njs_vm end = &s.start[s.size]; if (utf8 == NJS_STRING_UTF8) { - start = njs_string_offset(s.start, s.start + s.size, p); + start = (p < length) ? njs_string_offset(s.start, s.start + s.size, p) + : end; } else { start = &s.start[p]; diff -r 1c066436ec55 -r a0f5719c8d9a src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Fri Jun 11 15:15:48 2021 +0000 +++ b/src/test/njs_unit_test.c Fri Jun 11 15:47:37 2021 +0000 @@ -8861,6 +8861,9 @@ static njs_unit_test_t njs_test[] = { njs_str("'abc'.split(/abc/)"), njs_str(",") }, + { njs_str("('?'.repeat(32)).split(/./).length"), + njs_str("33") }, + { njs_str("'AbcDefGhi'.split(/([A-Z][a-z]+)/)"), njs_str(",Abc,,Def,,Ghi,") }, From xeioex at nginx.com Fri Jun 11 18:28:41 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 11 Jun 2021 18:28:41 +0000 Subject: [njs] Fixed use-of-uninitialized-value in njs_string_index(). Message-ID: details: https://hg.nginx.org/njs/rev/db7696b86a9c branches: changeset: 1659:db7696b86a9c user: Dmitry Volyntsev date: Fri Jun 11 18:28:21 2021 +0000 description: Fixed use-of-uninitialized-value in njs_string_index(). Found by MemorySanitizer. diffstat: src/njs_string.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r a0f5719c8d9a -r db7696b86a9c src/njs_string.c --- a/src/njs_string.c Fri Jun 11 15:47:37 2021 +0000 +++ b/src/njs_string.c Fri Jun 11 18:28:21 2021 +0000 @@ -2565,7 +2565,7 @@ njs_string_index(njs_string_prop_t *stri last = 0; index = 0; - if (string->length >= NJS_STRING_MAP_STRIDE) { + if (string->length > NJS_STRING_MAP_STRIDE) { end = string->start + string->size; map = njs_string_map_start(end); From tracey at archive.org Sat Jun 12 23:01:05 2021 From: tracey at archive.org (Tracey Jaquith) Date: Sat, 12 Jun 2021 16:01:05 -0700 Subject: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes In-Reply-To: <20210609121013.5tfxi2bao6rqkr7j@Romans-MacBook-Pro.local> References: <5da9c62fa61016600f2c.1622750089@vm-home1.archive.org> <20210609121013.5tfxi2bao6rqkr7j@Romans-MacBook-Pro.local> Message-ID: <23599EF3-9B38-4C47-B8B8-1F6EDB6C8006@archive.org> Hello Roman, > On Jun 9, 2021, at 5:10 AM, Roman Arutyunyan wrote: > > Hello Tracey. > > Thanks for your patch, it looks very interesting. Yay! :) > > On Thu, Jun 03, 2021 at 07:54:49PM +0000, Tracey Jaquith wrote: >> # HG changeset patch >> # User Tracey Jaquith >> # Date 1622678642 0 >> # Thu Jun 03 00:04:02 2021 +0000 >> # Node ID 5da9c62fa61016600f2c59982ae184e2811be427 >> # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 >> Add optional "&exact=1" CGI arg to show video between keyframes. > > I think this can be a directive (like mp4_buffer_size/mp4_max_buffer_size) > rather than an argument. Is it possible than we need both exact and non-exact > in the same location? It feels like we don't need to give clients control over > this. The exact mode introduces some overhead and should only be used when > needed. That?s a fine idea. So far, we?ve been using ?exact clipping? at archive.org both ways ? but there?s some interest from our TV team to just always do the ?exact? mode, anyway. So that could work for us. That something I should look into for a revised / 2nd patch, sounds like? > >> archive.org has been using mod_h264_streaming with an "&exact=1" patch from me since 2013. >> We just moved to nginx mp4 module and are using this patch. >> The technique is to find the keyframe just before the desired "start" time, and send >> that down the wire so video playback can start immediately. >> Next calculate how many video samples are between the keyframe and desired "start" time >> and update the STTS atom where those samples move the duration from (typically) 1001 to 1. >> This way, initial unwanted video frames play at ~1/30,000s -- so visually the >> video & audio start playing immediately. >> >> You can see an example before/after here (nginx binary built with mp4 module + patch): >> >> https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30 >> https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30&exact=1 >> >> Tested on linux and macosx. >> >> I realize two var declarations should style-wise get moved "up" in the lower hooked in method. >> However, to minimize the changeset/patch, I figured we should at least start here and see what you folks think. > > Syle-wise there are a few minor issues. We'll figure these out later. Perfect thanks. I figured _at least_ those var decls but am sure there are probably some more since I?m stepping into your house :) > > Below are my comments on ways to simplify the code. > >> (this is me: https://github.com/traceypooh ) >> >> diff -r 5f765427c17a -r 5da9c62fa610 src/http/modules/ngx_http_mp4_module.c >> --- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300 >> +++ b/src/http/modules/ngx_http_mp4_module.c Thu Jun 03 00:04:02 2021 +0000 >> @@ -2045,6 +2045,109 @@ >> u_char duration[4]; >> } ngx_mp4_stts_entry_t; >> >> +typedef struct { >> + uint32_t speedup_samples; >> + ngx_uint_t speedup_seconds; >> +} ngx_mp4_exact_t; >> + >> +static void >> +exact_video_adjustment(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_mp4_exact_t *exact) >> +{ >> + // will parse STTS -- time-to-sample atom >> + ngx_str_t value; >> + ngx_buf_t *stts_data; >> + ngx_buf_t *atom; >> + ngx_mp4_stts_entry_t *stts_entry, *stts_end; >> + uint32_t count, duration, j, n, sample_keyframe, sample_num; >> + uint64_t sample_time, seconds, start_seconds_closest_keyframe; >> + uint8_t is_keyframe; >> + >> + exact->speedup_samples = 0; >> + exact->speedup_seconds = 0; >> + >> + // if '&exact=1' CGI arg isn't present, do nothing >> + if (!(ngx_http_arg(mp4->request, (u_char *) "exact", 5, &value) == NGX_OK)) { >> + return; >> + } >> + >> + if (!trak->sync_samples_entries) { >> + // Highly unlikely video STSS got parsed and _every_ sample is a keyframe. >> + // However, if the case, we don't need to adjust the video at all. >> + return; >> + } >> + >> + // check HDLR atom to see if this trak is video or audio >> + atom = trak->out[NGX_HTTP_MP4_HDLR_ATOM].buf; >> + // 'vide' or 'soun' >> + if (!(atom->pos[16] == 'v' && >> + atom->pos[17] == 'i' && >> + atom->pos[18] == 'd' && >> + atom->pos[19] == 'e')) { >> + return; // do nothing if not video >> + } > > Do we really need to check this? If a track has an stss atom, this seems > enough to do the work. With the .mp4 that we make at archive.org , I am seeing STSS consistently for the audio track. That?s pointing out the start times of the audio samples, so that sort of makes sense to me. So I found we needed a way to do nothing for any audio tracks. I guess? we _could_ maybe run the same logic on any audio track(s) since, presumably, each audio sample is a ?keyframe? (and so theoretically we should do nothing below)? If you?d prefer something like that, I?d need to do a little digging/testing. > >> + stts_data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf; >> + stts_entry = (ngx_mp4_stts_entry_t *) stts_data->pos; >> + stts_end = (ngx_mp4_stts_entry_t *) stts_data->last; >> + >> + sample_num = 0; // they start at one >> + sample_time = 0; >> + start_seconds_closest_keyframe = 0; >> + while (stts_entry < stts_end) { >> + // STTS === time-to-sample atom >> + // each entry is 4B and [sample count][sample duration] (since durations can vary) >> + count = ngx_mp4_get_32value(stts_entry->count); >> + duration = ngx_mp4_get_32value(stts_entry->duration); >> + >> + for (j = 0; j < count; j++) { >> + sample_num++; >> + >> + // search STSS sync sample entries to see if this sample is a keyframe >> + is_keyframe = (trak->sync_samples_entries ? 0 : 1); > > Considering the above, trak->sync_samples_entries is always non-zero here. Oh, excellent eyes, thanks! I will update that to is_keyframe = 0; >> + for (n = 0; n < trak->sync_samples_entries; n++) { >> + // each one of this these are a video sample number keyframe >> + sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4)); >> + if (sample_keyframe == sample_num) { >> + is_keyframe = 1; >> + break; >> + } >> + if (sample_keyframe > sample_num) { >> + break; >> + } >> + } >> + >> + seconds = sample_time * 1000 / trak->timescale; >> + sample_time += duration; >> + >> + if (seconds > mp4->start) { >> + goto found; >> + } >> + >> + if (is_keyframe) { >> + start_seconds_closest_keyframe = seconds; >> + exact->speedup_samples = 0; >> + } else { >> + exact->speedup_samples++; >> + } >> + } >> + >> + stts_entry++; >> + } >> + >> + found: >> + >> + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, >> + "exact_video_adjustment() new keyframe start: %d, speedup first %d samples", >> + start_seconds_closest_keyframe, >> + exact->speedup_samples); >> + >> + // NOTE: begin 1 start position before keyframe to ensure first video frame emitted is always >> + // a keyframe >> + exact->speedup_seconds = mp4->start - start_seconds_closest_keyframe >> + - (start_seconds_closest_keyframe ? 1 : 0); >> +} >> + >> >> static ngx_int_t >> ngx_http_mp4_read_stts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) >> @@ -2164,10 +2267,14 @@ >> ngx_buf_t *data; >> ngx_uint_t start_sample, entries, start_sec; >> ngx_mp4_stts_entry_t *entry, *end; >> + ngx_mp4_exact_t exact; >> >> if (start) { >> start_sec = mp4->start; >> >> + exact_video_adjustment(mp4, trak, &exact); >> + start_sec -= exact.speedup_seconds; > > Here you find the closest key frame from the past and adjust start_sec to > match it. But what if we don't do this here, but continue with the original > start_sec. When we find the right (probably non-key) frame, we'll have its > number in start_sample. All we'll need to do at that point is to search for > the closest key frame to that sample number - just a simple loop. When we > find the number of samples into the past until the most recent key frame, > we'll just add one entry to the stts array with (this number, 1) and then > decrease start_sample accordingly. Sounds simpler. Ah, ooh, that?s very interesting! I like the sound of it. So we?d use trak->start_sample Sounds like, and go? either forwards or backwards in a simple loop over the trak->sync_samples_entries until we find the the keyframe entry number that is just less than or equal to the trak->start_sample Does that sound right? And I think we could be adding up to 300 STTS entries compared to the non-exact method (depending on video, but assuming up to 10s between keyframes and 30fps). > >> + >> ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, >> "mp4 stts crop start_time:%ui", start_sec); >> >> @@ -2230,6 +2337,42 @@ >> >> if (start) { >> ngx_mp4_set_32value(entry->count, count - rest); >> + >> + if (exact.speedup_samples) { >> + // We're going to prepend an entry with duration=1 for the frames we want to "not see". >> + // MOST of the time, we're taking a single element entry array and making it two. >> + uint32_t current_count = ngx_mp4_get_32value(entry->count); >> + ngx_mp4_stts_entry_t* entries_array = ngx_palloc(mp4->request->pool, >> + (1 + entries) * sizeof(ngx_mp4_stts_entry_t)); >> + if (entries_array == NULL) { >> + return NGX_ERROR; >> + } > > I believe there are situations when there's an entry from the past in the old > array, that can be reused. So this is indeed a _very_ interesting point! Most of the time, certainly for our created .mp4, we?re talking about constant frame rates for the video. (The audio is indeed often a huge list of entries w/ minor variance in their durations). So for the video, I?m mostly only ever seeing a single entry list with duration ?1001? (For the 29.97 fps typical TV we see in US where it?s 30000 / 1001 => 29.97). I did start trying to make the logical list split anywhere it might be found for 2+ entries, but it _really_ was making my head hurt since there were so many computations to do while trying to figure out where all the frames were, and where we might be splitting. In the end, I found I have almost entirely single-entry arrays, splitting into two. (Though the current patch doesn?t care what the number of array elements is ? it just prepends a new entry (of duration 1 instead of typical 1001) via a logical `realloc()`) If you?d prefer something for 2+ entries case, that tries to see if we?re dropping enough entries from the front ? and to just reuse one (and not realloc ? though we?d need a realloc case for corner cases anyway, _i think_) then perhaps I could use some help or thoughts on that since it was getting super complicated when I was trying to diagram it out and was fretting and nearly paralyzed with how to proceed cleanly and safely :-p Thanks so much for the detailed feedback and time for your thoughts! Super appreciated and I?m very upbeat about this all. -Tracey > >> + ngx_copy(&(entries_array[1]), entry, entries * sizeof(ngx_mp4_stts_entry_t)); >> + >> + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, >> + "exact split in 2 video STTS entry from count:%d", current_count); >> + >> + if (current_count <= exact.speedup_samples) >> + return NGX_ERROR; >> + >> + entry = &(entries_array[1]); >> + ngx_mp4_set_32value(entry->count, current_count - exact.speedup_samples); >> + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, >> + "exact split new[1]: count:%d duration:%d", >> + ngx_mp4_get_32value(entry->count), >> + ngx_mp4_get_32value(entry->duration)); >> + entry--; >> + ngx_mp4_set_32value(entry->count, exact.speedup_samples); >> + ngx_mp4_set_32value(entry->duration, 1); >> + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, >> + "exact split new[0]: count:%d duration:1", >> + ngx_mp4_get_32value(entry->count)); >> + >> + data->last = (u_char *) (entry + 2); >> + >> + entries++; >> + } >> + >> data->pos = (u_char *) entry; >> trak->time_to_sample_entries = entries; >> trak->start_sample = start_sample; >> _______________________________________________ >> nginx-devel mailing list >> nginx-devel at nginx.org >> http://mailman.nginx.org/mailman/listinfo/nginx-devel > > -- > Roman Arutyunyan > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > http://mailman.nginx.org/mailman/listinfo/nginx-devel -Tracey @tracey_pooh TV Architect https://archive.org/tv -------------- next part -------------- An HTML attachment was scrubbed... URL: From tracey at archive.org Sun Jun 13 09:53:43 2021 From: tracey at archive.org (Tracey Jaquith) Date: Sun, 13 Jun 2021 02:53:43 -0700 Subject: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes In-Reply-To: <23599EF3-9B38-4C47-B8B8-1F6EDB6C8006@archive.org> References: <5da9c62fa61016600f2c.1622750089@vm-home1.archive.org> <20210609121013.5tfxi2bao6rqkr7j@Romans-MacBook-Pro.local> <23599EF3-9B38-4C47-B8B8-1F6EDB6C8006@archive.org> Message-ID: <7EC61C38-F1E1-4BDF-855C-8A83AFFA2A47@archive.org> Roman - genius that works! :) I?ll formalize this towards using an nginx config var exact setting and some minor cleanup, but here?s the new smaller patch, logically, with your idea: Sample item working with all sort of clip testing, and a start/end with prior patch and this patch yielded bitwise identical emitted mp4 :) Method: ngx_http_mp4_crop_stts_data() if (start) { ngx_mp4_set_32value(entry->count, count - rest); data->pos = (u_char *) entry; trak->time_to_sample_entries = entries; trak->start_sample = start_sample; ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "start_sample:%ui, new count:%uD", trak->start_sample, count - rest); + // xxx only do this if nginx mp4 config exact setting is set + uint32_t n, speedup_samples; + ngx_uint_t sample_keyframe, start_sample_exact; + + start_sample_exact = trak->start_sample; + for (n = 0; n < trak->sync_samples_entries; n++) { + // each element of array is the sample number of a keyframe + // sync samples starts from 1 -- so subtract 1 + sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4)) - 1; + if (sample_keyframe <= trak->start_sample) { + start_sample_exact = sample_keyframe; + } + if (sample_keyframe >= trak->start_sample) { + break; + } + } + + if (start_sample_exact < trak->start_sample) { + // We're going to prepend an entry with duration=1 for the frames we want to "not see". + // MOST of the time (eg: constant video framerate), + // we're taking a single element entry array and making it two. + speedup_samples = trak->start_sample - start_sample_exact; + + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact trak start_sample move %l to %l (speed up %d samples)\n", + trak->start_sample, start_sample_exact, speedup_samples); + + uint32_t current_count = ngx_mp4_get_32value(entry->count); + ngx_mp4_stts_entry_t* entries_array = ngx_palloc(mp4->request->pool, + (1 + entries) * sizeof(ngx_mp4_stts_entry_t)); + if (entries_array == NULL) { + return NGX_ERROR; + } + ngx_copy(&(entries_array[1]), entry, entries * sizeof(ngx_mp4_stts_entry_t)); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split in 2 video STTS entry from count:%d", current_count); + + if (current_count <= speedup_samples) { + return NGX_ERROR; + } + + entry = &(entries_array[1]); + ngx_mp4_set_32value(entry->count, current_count - speedup_samples); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split new[1]: count:%d duration:%d", + ngx_mp4_get_32value(entry->count), + ngx_mp4_get_32value(entry->duration)); + entry--; + ngx_mp4_set_32value(entry->count, speedup_samples); + ngx_mp4_set_32value(entry->duration, 1); + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split new[0]: count:%d duration:1", + ngx_mp4_get_32value(entry->count)); + + data->pos = (u_char *) entry; + trak->time_to_sample_entries++; + trak->start_sample = start_sample_exact; + data->last = (u_char *) (entry + trak->time_to_sample_entries); + } } else { ngx_mp4_set_32value(entry->count, rest); data->last = (u_char *) (entry + 1); trak->time_to_sample_entries -= entries - 1; trak->end_sample = trak->start_sample + start_sample; ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "end_sample:%ui, new count:%uD", trak->end_sample, rest); } Thanks for that great insight! -Tracey > On Jun 12, 2021, at 4:01 PM, Tracey Jaquith wrote: > > Hello Roman, > >> On Jun 9, 2021, at 5:10 AM, Roman Arutyunyan > wrote: >> >> Hello Tracey. >> >> Thanks for your patch, it looks very interesting. > > Yay! :) > >> >> On Thu, Jun 03, 2021 at 07:54:49PM +0000, Tracey Jaquith wrote: >>> # HG changeset patch >>> # User Tracey Jaquith > >>> # Date 1622678642 0 >>> # Thu Jun 03 00:04:02 2021 +0000 >>> # Node ID 5da9c62fa61016600f2c59982ae184e2811be427 >>> # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 >>> Add optional "&exact=1" CGI arg to show video between keyframes. >> >> I think this can be a directive (like mp4_buffer_size/mp4_max_buffer_size) >> rather than an argument. Is it possible than we need both exact and non-exact >> in the same location? It feels like we don't need to give clients control over >> this. The exact mode introduces some overhead and should only be used when >> needed. > > That?s a fine idea. > So far, we?ve been using ?exact clipping? at archive.org both ways ? but there?s some interest from our TV team to just always do the ?exact? mode, anyway. > So that could work for us. > That something I should look into for a revised / 2nd patch, sounds like? > >> >>> archive.org has been using mod_h264_streaming with an "&exact=1" patch from me since 2013. >>> We just moved to nginx mp4 module and are using this patch. >>> The technique is to find the keyframe just before the desired "start" time, and send >>> that down the wire so video playback can start immediately. >>> Next calculate how many video samples are between the keyframe and desired "start" time >>> and update the STTS atom where those samples move the duration from (typically) 1001 to 1. >>> This way, initial unwanted video frames play at ~1/30,000s -- so visually the >>> video & audio start playing immediately. >>> >>> You can see an example before/after here (nginx binary built with mp4 module + patch): >>> >>> https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30 >>> https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30&exact=1 >>> >>> Tested on linux and macosx. >>> >>> I realize two var declarations should style-wise get moved "up" in the lower hooked in method. >>> However, to minimize the changeset/patch, I figured we should at least start here and see what you folks think. >> >> Syle-wise there are a few minor issues. We'll figure these out later. > > Perfect thanks. I figured _at least_ those var decls but am sure there are probably some more since I?m stepping into your house :) > >> >> Below are my comments on ways to simplify the code. >> >>> (this is me: https://github.com/traceypooh ) >>> >>> diff -r 5f765427c17a -r 5da9c62fa610 src/http/modules/ngx_http_mp4_module.c >>> --- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300 >>> +++ b/src/http/modules/ngx_http_mp4_module.c Thu Jun 03 00:04:02 2021 +0000 >>> @@ -2045,6 +2045,109 @@ >>> u_char duration[4]; >>> } ngx_mp4_stts_entry_t; >>> >>> +typedef struct { >>> + uint32_t speedup_samples; >>> + ngx_uint_t speedup_seconds; >>> +} ngx_mp4_exact_t; >>> + >>> +static void >>> +exact_video_adjustment(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_mp4_exact_t *exact) >>> +{ >>> + // will parse STTS -- time-to-sample atom >>> + ngx_str_t value; >>> + ngx_buf_t *stts_data; >>> + ngx_buf_t *atom; >>> + ngx_mp4_stts_entry_t *stts_entry, *stts_end; >>> + uint32_t count, duration, j, n, sample_keyframe, sample_num; >>> + uint64_t sample_time, seconds, start_seconds_closest_keyframe; >>> + uint8_t is_keyframe; >>> + >>> + exact->speedup_samples = 0; >>> + exact->speedup_seconds = 0; >>> + >>> + // if '&exact=1' CGI arg isn't present, do nothing >>> + if (!(ngx_http_arg(mp4->request, (u_char *) "exact", 5, &value) == NGX_OK)) { >>> + return; >>> + } >>> + >>> + if (!trak->sync_samples_entries) { >>> + // Highly unlikely video STSS got parsed and _every_ sample is a keyframe. >>> + // However, if the case, we don't need to adjust the video at all. >>> + return; >>> + } >>> + >>> + // check HDLR atom to see if this trak is video or audio >>> + atom = trak->out[NGX_HTTP_MP4_HDLR_ATOM].buf; >>> + // 'vide' or 'soun' >>> + if (!(atom->pos[16] == 'v' && >>> + atom->pos[17] == 'i' && >>> + atom->pos[18] == 'd' && >>> + atom->pos[19] == 'e')) { >>> + return; // do nothing if not video >>> + } >> >> Do we really need to check this? If a track has an stss atom, this seems >> enough to do the work. > > With the .mp4 that we make at archive.org , I am seeing STSS consistently for the audio track. > That?s pointing out the start times of the audio samples, so that sort of makes sense to me. > So I found we needed a way to do nothing for any audio tracks. > > I guess? we _could_ maybe run the same logic on any audio track(s) since, presumably, > each audio sample is a ?keyframe? (and so theoretically we should do nothing below)? > If you?d prefer something like that, I?d need to do a little digging/testing. > > >> >>> + stts_data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf; >>> + stts_entry = (ngx_mp4_stts_entry_t *) stts_data->pos; >>> + stts_end = (ngx_mp4_stts_entry_t *) stts_data->last; >>> + >>> + sample_num = 0; // they start at one >>> + sample_time = 0; >>> + start_seconds_closest_keyframe = 0; >>> + while (stts_entry < stts_end) { >>> + // STTS === time-to-sample atom >>> + // each entry is 4B and [sample count][sample duration] (since durations can vary) >>> + count = ngx_mp4_get_32value(stts_entry->count); >>> + duration = ngx_mp4_get_32value(stts_entry->duration); >>> + >>> + for (j = 0; j < count; j++) { >>> + sample_num++; >>> + >>> + // search STSS sync sample entries to see if this sample is a keyframe >>> + is_keyframe = (trak->sync_samples_entries ? 0 : 1); >> >> Considering the above, trak->sync_samples_entries is always non-zero here. > > Oh, excellent eyes, thanks! > I will update that to > is_keyframe = 0; > > >>> + for (n = 0; n < trak->sync_samples_entries; n++) { >>> + // each one of this these are a video sample number keyframe >>> + sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4)); >>> + if (sample_keyframe == sample_num) { >>> + is_keyframe = 1; >>> + break; >>> + } >>> + if (sample_keyframe > sample_num) { >>> + break; >>> + } >>> + } >>> + >>> + seconds = sample_time * 1000 / trak->timescale; >>> + sample_time += duration; >>> + >>> + if (seconds > mp4->start) { >>> + goto found; >>> + } >>> + >>> + if (is_keyframe) { >>> + start_seconds_closest_keyframe = seconds; >>> + exact->speedup_samples = 0; >>> + } else { >>> + exact->speedup_samples++; >>> + } >>> + } >>> + >>> + stts_entry++; >>> + } >>> + >>> + found: >>> + >>> + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, >>> + "exact_video_adjustment() new keyframe start: %d, speedup first %d samples", >>> + start_seconds_closest_keyframe, >>> + exact->speedup_samples); >>> + >>> + // NOTE: begin 1 start position before keyframe to ensure first video frame emitted is always >>> + // a keyframe >>> + exact->speedup_seconds = mp4->start - start_seconds_closest_keyframe >>> + - (start_seconds_closest_keyframe ? 1 : 0); >>> +} >>> + >>> >>> static ngx_int_t >>> ngx_http_mp4_read_stts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) >>> @@ -2164,10 +2267,14 @@ >>> ngx_buf_t *data; >>> ngx_uint_t start_sample, entries, start_sec; >>> ngx_mp4_stts_entry_t *entry, *end; >>> + ngx_mp4_exact_t exact; >>> >>> if (start) { >>> start_sec = mp4->start; >>> >>> + exact_video_adjustment(mp4, trak, &exact); >>> + start_sec -= exact.speedup_seconds; >> >> Here you find the closest key frame from the past and adjust start_sec to >> match it. But what if we don't do this here, but continue with the original >> start_sec. When we find the right (probably non-key) frame, we'll have its >> number in start_sample. All we'll need to do at that point is to search for >> the closest key frame to that sample number - just a simple loop. When we >> find the number of samples into the past until the most recent key frame, >> we'll just add one entry to the stts array with (this number, 1) and then >> decrease start_sample accordingly. Sounds simpler. > > Ah, ooh, that?s very interesting! I like the sound of it. > So we?d use > trak->start_sample > Sounds like, and go? either forwards or backwards in a simple loop over the > trak->sync_samples_entries > until we find the the keyframe entry number that is just less than or equal to the > trak->start_sample > Does that sound right? > > And I think we could be adding up to 300 STTS entries compared to the non-exact method > (depending on video, but assuming up to 10s between keyframes and 30fps). > > >> >>> + >>> ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, >>> "mp4 stts crop start_time:%ui", start_sec); >>> >>> @@ -2230,6 +2337,42 @@ >>> >>> if (start) { >>> ngx_mp4_set_32value(entry->count, count - rest); >>> + >>> + if (exact.speedup_samples) { >>> + // We're going to prepend an entry with duration=1 for the frames we want to "not see". >>> + // MOST of the time, we're taking a single element entry array and making it two. >>> + uint32_t current_count = ngx_mp4_get_32value(entry->count); >>> + ngx_mp4_stts_entry_t* entries_array = ngx_palloc(mp4->request->pool, >>> + (1 + entries) * sizeof(ngx_mp4_stts_entry_t)); >>> + if (entries_array == NULL) { >>> + return NGX_ERROR; >>> + } >> >> I believe there are situations when there's an entry from the past in the old >> array, that can be reused. > > So this is indeed a _very_ interesting point! > Most of the time, certainly for our created .mp4, we?re talking about constant frame rates for the video. > (The audio is indeed often a huge list of entries w/ minor variance in their durations). > So for the video, I?m mostly only ever seeing a single entry list with duration ?1001? > (For the 29.97 fps typical TV we see in US where it?s 30000 / 1001 => 29.97). > > I did start trying to make the logical list split anywhere it might be found for 2+ entries, > but it _really_ was making my head hurt since there were so many computations to do > while trying to figure out where all the frames were, and where we might be splitting. > > In the end, I found I have almost entirely single-entry arrays, splitting into two. > (Though the current patch doesn?t care what the number of array elements is ? it just > prepends a new entry (of duration 1 instead of typical 1001) via a logical `realloc()`) > > If you?d prefer something for 2+ entries case, that tries to see if we?re dropping enough entries from > the front ? and to just reuse one (and not realloc ? though we?d need a realloc case for > corner cases anyway, _i think_) then perhaps I could use some help or thoughts on that since > it was getting super complicated when I was trying to diagram it out and was fretting and > nearly paralyzed with how to proceed cleanly and safely :-p > > Thanks so much for the detailed feedback and time for your thoughts! > Super appreciated and I?m very upbeat about this all. > > -Tracey > >> >>> + ngx_copy(&(entries_array[1]), entry, entries * sizeof(ngx_mp4_stts_entry_t)); >>> + >>> + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, >>> + "exact split in 2 video STTS entry from count:%d", current_count); >>> + >>> + if (current_count <= exact.speedup_samples) >>> + return NGX_ERROR; >>> + >>> + entry = &(entries_array[1]); >>> + ngx_mp4_set_32value(entry->count, current_count - exact.speedup_samples); >>> + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, >>> + "exact split new[1]: count:%d duration:%d", >>> + ngx_mp4_get_32value(entry->count), >>> + ngx_mp4_get_32value(entry->duration)); >>> + entry--; >>> + ngx_mp4_set_32value(entry->count, exact.speedup_samples); >>> + ngx_mp4_set_32value(entry->duration, 1); >>> + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, >>> + "exact split new[0]: count:%d duration:1", >>> + ngx_mp4_get_32value(entry->count)); >>> + >>> + data->last = (u_char *) (entry + 2); >>> + >>> + entries++; >>> + } >>> + >>> data->pos = (u_char *) entry; >>> trak->time_to_sample_entries = entries; >>> trak->start_sample = start_sample; >>> _______________________________________________ >>> nginx-devel mailing list >>> nginx-devel at nginx.org >>> http://mailman.nginx.org/mailman/listinfo/nginx-devel >> >> -- >> Roman Arutyunyan >> _______________________________________________ >> nginx-devel mailing list >> nginx-devel at nginx.org >> http://mailman.nginx.org/mailman/listinfo/nginx-devel > > -Tracey > @tracey_pooh > TV Architect https://archive.org/tv -Tracey @tracey_pooh TV Architect https://archive.org/tv -------------- next part -------------- An HTML attachment was scrubbed... URL: From tung.nt.pham90 at gmail.com Mon Jun 14 13:39:10 2021 From: tung.nt.pham90 at gmail.com (tung pham) Date: Mon, 14 Jun 2021 14:39:10 +0100 Subject: Nginx-tests are failing for version 1.19.5 and prior. Message-ID: Hi all, We have recently run a full test using the nginx-tests tip for Nginx 1.19.0, 1.19.5, 1.19.10 and 1.20.0 and the tests are reported failing for 1.19.0 and 1.19.5. Below is a summary of the test report: Test Summary Report ------------------- tests/nginx-tests/http_keepalive.t (Wstat: 512 Tests: 0 Failed: 0) Non-zero exit status: 2 Parse errors: No plan found in TAP output tests/nginx-tests/http_uri.t (Wstat: 1024 Tests: 19 Failed: 4) Failed tests: 11-12, 15-16 Non-zero exit status: 4 tests/nginx-tests/proxy_cookie_flags.t (Wstat: 512 Tests: 0 Failed: 0) Non-zero exit status: 2 Parse errors: No plan found in TAP output tests/nginx-tests/proxy_next_upstream.t (Wstat: 256 Tests: 10 Failed: 1) Failed test: 8 Non-zero exit status: 1 tests/nginx-tests/upstream_keepalive.t (Wstat: 512 Tests: 0 Failed: 0) Non-zero exit status: 2 Parse errors: No plan found in TAP output Files=401, Tests=2106, 228 wallclock secs ( 0.71 usr 0.43 sys + 26.16 cusr 6.00 csys = 33.30 CPU) Result: FAIL These tests were recently activated and as far as I understand, all versions of Nginx should use the tip version. Should these tests be excluded/disabled from the tip tag for version 1.19.5 and prior until the failures are addressed. Apologies if we are the only ones that get the failures. P/S: These are run with nginx alone without any of our third party modules. Many thanks, Tung Pham -------------- next part -------------- An HTML attachment was scrubbed... URL: From lcuminato at gmail.com Mon Jun 14 15:08:34 2021 From: lcuminato at gmail.com (Lucas Cuminato) Date: Mon, 14 Jun 2021 10:08:34 -0500 Subject: [nginx-quic] Message-ID: Hello, Not sure If this is a bug in nginx-quic or if I'm not configuring it correctly but when trying to use nginx-quic with the following settings. stream { server { listen 5555 quic reuseport; ssl_session_cache off; ssl_client_certificate ca.pem ssl_verify_client on; ssl_session_tickets off; ssl_certificate cert.pem ssl_certificate_key key.pem; ssl_protocols TLSv1.3; } } and using a standalone application that uses ngtcp2 to try to connect to nginx-quic, I get a TLS alert saying that "No application protocol". I've tracked this down and it seems like nginx-quic is not setting any ALPN for the SSL context when using QUIC as a stream (in ngx_stream_ssl_module.c). It does it set it when using QUIC as HTTP (in ngx_http_ssl_module.c). Now, I believe ALPN is mandatory for QUIC according to the QUIC-TRANSPORT draft, so this might be a bug. By copying the code done in ngx_http_ssl_module.c for setting the ALPN and using it in ngx_stream_ssl_module.c, I was able to make my standalone app connect and transfer data, but not sure if this is the right fix. R, Lucas. -------------- next part -------------- An HTML attachment was scrubbed... URL: From vl at nginx.com Mon Jun 14 16:35:06 2021 From: vl at nginx.com (Vladimir Homutov) Date: Mon, 14 Jun 2021 19:35:06 +0300 Subject: [nginx-quic] In-Reply-To: References: Message-ID: <02933390-dbeb-7d80-e3a4-fa4ffe5bf45b@nginx.com> 14.06.2021 18:08, Lucas Cuminato ?????: > Hello, > > Not sure If this is a bug in nginx-quic or if I'm not configuring > it?correctly but when trying to use nginx-quic with the following settings. > > stream { > ? ? server { > ? ? ? ? listen 5555 quic reuseport; > ? ? ? ? ssl_session_cache off; > ? ? ? ? ssl_client_certificate ca.pem > ? ? ? ? ssl_verify_client on; > ? ? ? ? ssl_session_tickets off; > ? ? ? ? ssl_certificate? ? ? ? ?cert.pem > ? ? ? ? ssl_certificate_key ? ?key.pem; > ? ? ? ? ssl_protocols ? ? ? TLSv1.3; > ? ? } > } > > and using a standalone application that uses ngtcp2 to try to connect to > nginx-quic, I get a TLS alert saying that "No application protocol". > I've tracked this down and it seems like nginx-quic is not setting any > ALPN for the SSL context when using QUIC as a stream (in > ngx_stream_ssl_module.c). > It does it set it when using QUIC as HTTP (in?ngx_http_ssl_module.c). > Now, I believe ALPN is mandatory for QUIC according to the > QUIC-TRANSPORT draft, so this might be a bug. > By copying the code done in?ngx_http_ssl_module.c for setting the ALPN > and using it in?ngx_stream_ssl_module.c, I was able to make my > standalone app connect and transfer data, but not sure > if this is the right fix. > > R, > Lucas. > Hello, this is expected with stream module. ALPN is required, but is not clear what protocol (http3? other protocol over quic?) is going to be used. Can you please elaborate your use case? What are you going to achieve? Also, the suggested configuration is not going to work, since you don't have any content handling module (i.e. proxy_pass or return). From lcuminato at gmail.com Mon Jun 14 16:43:58 2021 From: lcuminato at gmail.com (Lucas Cuminato) Date: Mon, 14 Jun 2021 11:43:58 -0500 Subject: [nginx-quic] In-Reply-To: <02933390-dbeb-7d80-e3a4-fa4ffe5bf45b@nginx.com> References: <02933390-dbeb-7d80-e3a4-fa4ffe5bf45b@nginx.com> Message-ID: Hi, Vladimir, thanks for replying. I'm not using any protocol over QUIC, just using QUIC to send/receive raw data to/from my application and the server, and having nginx proxy it to a TCP server. I do have a proxy_pass configured in my setup. I just omitted for simplicity. R, Lucas. On Mon, Jun 14, 2021 at 11:35 AM Vladimir Homutov wrote: > 14.06.2021 18:08, Lucas Cuminato ?????: > > Hello, > > > > Not sure If this is a bug in nginx-quic or if I'm not configuring > > it correctly but when trying to use nginx-quic with the following > settings. > > > > stream { > > server { > > listen 5555 quic reuseport; > > ssl_session_cache off; > > ssl_client_certificate ca.pem > > ssl_verify_client on; > > ssl_session_tickets off; > > ssl_certificate cert.pem > > ssl_certificate_key key.pem; > > ssl_protocols TLSv1.3; > > } > > } > > > > and using a standalone application that uses ngtcp2 to try to connect to > > nginx-quic, I get a TLS alert saying that "No application protocol". > > I've tracked this down and it seems like nginx-quic is not setting any > > ALPN for the SSL context when using QUIC as a stream (in > > ngx_stream_ssl_module.c). > > It does it set it when using QUIC as HTTP (in ngx_http_ssl_module.c). > > Now, I believe ALPN is mandatory for QUIC according to the > > QUIC-TRANSPORT draft, so this might be a bug. > > By copying the code done in ngx_http_ssl_module.c for setting the ALPN > > and using it in ngx_stream_ssl_module.c, I was able to make my > > standalone app connect and transfer data, but not sure > > if this is the right fix. > > > > R, > > Lucas. > > > Hello, > this is expected with stream module. > ALPN is required, but is not clear what protocol (http3? other protocol > over quic?) is going to be used. > Can you please elaborate your use case? What are you going to achieve? > Also, the suggested configuration is not going to work, since you don't > have any content handling module (i.e. proxy_pass or return). > > > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > http://mailman.nginx.org/mailman/listinfo/nginx-devel -------------- next part -------------- An HTML attachment was scrubbed... URL: From vl at nginx.com Mon Jun 14 17:00:25 2021 From: vl at nginx.com (Vladimir Homutov) Date: Mon, 14 Jun 2021 20:00:25 +0300 Subject: [nginx-quic] In-Reply-To: References: <02933390-dbeb-7d80-e3a4-fa4ffe5bf45b@nginx.com> Message-ID: 14.06.2021 19:43, Lucas Cuminato ?????: > Hi, Vladimir, thanks for replying. > > I'm not using any protocol over QUIC, just using QUIC to send/receive > raw data to/from my application and the server, and having nginx proxy > it to a TCP server. > I do have a proxy_pass configured in my setup. I just omitted for > simplicity. > > R, > Lucas. Ok, so you have custom backend that knows what to do with QUIC streams? And you backend is TCP-based? Sounds quite interesting. Or does it deal with single stream only? Anyway, right now it fails at ALPN stage. Probably, in future, we may introduce some configuration directive to control it. It is not yet absolutely clear how the stream module should deal with quic. Yoy may want to try to copy the code wich sets ALPN callback from http_quic module and provides some meaningful value for protocol. > > > On Mon, Jun 14, 2021 at 11:35 AM Vladimir Homutov > wrote: > > 14.06.2021 18:08, Lucas Cuminato ?????: > > Hello, > > > > Not sure If this is a bug in nginx-quic or if I'm not configuring > > it?correctly but when trying to use nginx-quic with the following > settings. > > > > stream { > >? ? ? server { > >? ? ? ? ? listen 5555 quic reuseport; > >? ? ? ? ? ssl_session_cache off; > >? ? ? ? ? ssl_client_certificate ca.pem > >? ? ? ? ? ssl_verify_client on; > >? ? ? ? ? ssl_session_tickets off; > >? ? ? ? ? ssl_certificate? ? ? ? ?cert.pem > >? ? ? ? ? ssl_certificate_key ? ?key.pem; > >? ? ? ? ? ssl_protocols ? ? ? TLSv1.3; > >? ? ? } > > } > > > > and using a standalone application that uses ngtcp2 to try to > connect to > > nginx-quic, I get a TLS alert saying that "No application protocol". > > I've tracked this down and it seems like nginx-quic is not > setting any > > ALPN for the SSL context when using QUIC as a stream (in > > ngx_stream_ssl_module.c). > > It does it set it when using QUIC as HTTP > (in?ngx_http_ssl_module.c). > > Now, I believe ALPN is mandatory for QUIC according to the > > QUIC-TRANSPORT draft, so this might be a bug. > > By copying the code done in?ngx_http_ssl_module.c for setting the > ALPN > > and using it in?ngx_stream_ssl_module.c, I was able to make my > > standalone app connect and transfer data, but not sure > > if this is the right fix. > > > > R, > > Lucas. > > > Hello, > this is expected with stream module. > ALPN is required, but is not clear what protocol (http3? other protocol > over quic?) is going to be used. > Can you please elaborate your use case? What are you going to achieve? > Also, the suggested configuration is not going to work, since you don't > have any content handling module (i.e. proxy_pass or return). > > From lcuminato at gmail.com Mon Jun 14 17:18:15 2021 From: lcuminato at gmail.com (Lucas Cuminato) Date: Mon, 14 Jun 2021 12:18:15 -0500 Subject: [nginx-quic] In-Reply-To: References: <02933390-dbeb-7d80-e3a4-fa4ffe5bf45b@nginx.com> Message-ID: I'm using a single bidirectional stream, so my backend is just a simple TCP server that understands the data that is sent to it. I already tried setting the ALPN in the stream module and it actually worked. I was just not sure if that was the right thing to do. But what you proposed is best, I might try adding a custom directive that can control what to set the ALPN to. Thanks for the help. R, Lucas. On Mon, Jun 14, 2021 at 12:00 PM Vladimir Homutov wrote: > 14.06.2021 19:43, Lucas Cuminato ?????: > > Hi, Vladimir, thanks for replying. > > > > I'm not using any protocol over QUIC, just using QUIC to send/receive > > raw data to/from my application and the server, and having nginx proxy > > it to a TCP server. > > I do have a proxy_pass configured in my setup. I just omitted for > > simplicity. > > > > R, > > Lucas. > > Ok, so you have custom backend that knows what to do with QUIC streams? > And you backend is TCP-based? Sounds quite interesting. Or does it deal > with single stream only? > > Anyway, right now it fails at ALPN stage. Probably, in future, we may > introduce some configuration directive to control it. It is not yet > absolutely clear how the stream module should deal with quic. > > Yoy may want to try to copy the code wich sets ALPN callback from > http_quic module and provides some meaningful value for protocol. > > > > > > > On Mon, Jun 14, 2021 at 11:35 AM Vladimir Homutov > > wrote: > > > > 14.06.2021 18:08, Lucas Cuminato ?????: > > > Hello, > > > > > > Not sure If this is a bug in nginx-quic or if I'm not configuring > > > it correctly but when trying to use nginx-quic with the following > > settings. > > > > > > stream { > > > server { > > > listen 5555 quic reuseport; > > > ssl_session_cache off; > > > ssl_client_certificate ca.pem > > > ssl_verify_client on; > > > ssl_session_tickets off; > > > ssl_certificate cert.pem > > > ssl_certificate_key key.pem; > > > ssl_protocols TLSv1.3; > > > } > > > } > > > > > > and using a standalone application that uses ngtcp2 to try to > > connect to > > > nginx-quic, I get a TLS alert saying that "No application > protocol". > > > I've tracked this down and it seems like nginx-quic is not > > setting any > > > ALPN for the SSL context when using QUIC as a stream (in > > > ngx_stream_ssl_module.c). > > > It does it set it when using QUIC as HTTP > > (in ngx_http_ssl_module.c). > > > Now, I believe ALPN is mandatory for QUIC according to the > > > QUIC-TRANSPORT draft, so this might be a bug. > > > By copying the code done in ngx_http_ssl_module.c for setting the > > ALPN > > > and using it in ngx_stream_ssl_module.c, I was able to make my > > > standalone app connect and transfer data, but not sure > > > if this is the right fix. > > > > > > R, > > > Lucas. > > > > > Hello, > > this is expected with stream module. > > ALPN is required, but is not clear what protocol (http3? other > protocol > > over quic?) is going to be used. > > Can you please elaborate your use case? What are you going to > achieve? > > Also, the suggested configuration is not going to work, since you > don't > > have any content handling module (i.e. proxy_pass or return). > > > > > > > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > http://mailman.nginx.org/mailman/listinfo/nginx-devel -------------- next part -------------- An HTML attachment was scrubbed... URL: From mdounin at mdounin.ru Mon Jun 14 19:41:02 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 14 Jun 2021 22:41:02 +0300 Subject: Nginx-tests are failing for version 1.19.5 and prior. In-Reply-To: References: Message-ID: Hello! On Mon, Jun 14, 2021 at 02:39:10PM +0100, tung pham wrote: > We have recently run a full test using the nginx-tests tip for Nginx > 1.19.0, 1.19.5, 1.19.10 and 1.20.0 and the tests are reported failing for > 1.19.0 and 1.19.5. Below is a summary of the test report: [...] > These tests were recently activated and as far as I understand, all > versions of Nginx should use the tip version. Should these tests be > excluded/disabled from the tip tag for version 1.19.5 and prior until the > failures are addressed. Tests are expected to work on the supported versions, that is, the last mainline version (1.21.0 now, see http://nginx.org/en/download.html) and the last stable version (1.20.1 now). The 1.19.* versions are no longer supported, and relevant checks were recently removed from the test suite, see 5ac6efbe5552 (http://hg.nginx.org/nginx-tests/rev/5ac6efbe5552). -- Maxim Dounin http://mdounin.ru/ From tung.nt.pham90 at gmail.com Mon Jun 14 21:47:44 2021 From: tung.nt.pham90 at gmail.com (tung pham) Date: Mon, 14 Jun 2021 22:47:44 +0100 Subject: Nginx-tests are failing for version 1.19.5 and prior. In-Reply-To: References: Message-ID: Thanks Maxim. I was a bit confused before since there were checks for 1.17.* versions and I had expected them to be removed long ago, not now together with the more recent 1.19.* versions. If the purpose of the test is always for the mainline version then I will look to remove the 1.19.* from our test and advise our customer accordingly. Kind regards, Tung On Mon, 14 Jun 2021, 20:41 Maxim Dounin, wrote: > Hello! > > On Mon, Jun 14, 2021 at 02:39:10PM +0100, tung pham wrote: > > > We have recently run a full test using the nginx-tests tip for Nginx > > 1.19.0, 1.19.5, 1.19.10 and 1.20.0 and the tests are reported failing for > > 1.19.0 and 1.19.5. Below is a summary of the test report: > > [...] > > > These tests were recently activated and as far as I understand, all > > versions of Nginx should use the tip version. Should these tests be > > excluded/disabled from the tip tag for version 1.19.5 and prior until the > > failures are addressed. > > Tests are expected to work on the supported versions, that is, > the last mainline version (1.21.0 now, see > http://nginx.org/en/download.html) and the last stable version > (1.20.1 now). The 1.19.* versions are no longer supported, and > relevant checks were recently removed from the test suite, see > 5ac6efbe5552 (http://hg.nginx.org/nginx-tests/rev/5ac6efbe5552). > > -- > Maxim Dounin > http://mdounin.ru/ > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > http://mailman.nginx.org/mailman/listinfo/nginx-devel > -------------- next part -------------- An HTML attachment was scrubbed... URL: From xeioex at nginx.com Tue Jun 15 12:54:34 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 15 Jun 2021 12:54:34 +0000 Subject: [njs] Added forgotten paragraph section for 0.5.2. Message-ID: details: https://hg.nginx.org/njs/rev/0ee4118d3cf9 branches: changeset: 1660:0ee4118d3cf9 user: Dmitry Volyntsev date: Tue Jun 15 11:54:03 2021 +0000 description: Added forgotten paragraph section for 0.5.2. diffstat: CHANGES | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r db7696b86a9c -r 0ee4118d3cf9 CHANGES --- a/CHANGES Fri Jun 11 18:28:21 2021 +0000 +++ b/CHANGES Tue Jun 15 11:54:03 2021 +0000 @@ -13,6 +13,8 @@ Changes with njs 0.5.2 *) Feature: introduced the "status" property for stream session object. + Core: + *) Feature: added njs.on('exit') callback support. *) Bugfix: fixed property descriptor reuse for not extensible From xeioex at nginx.com Tue Jun 15 12:54:36 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 15 Jun 2021 12:54:36 +0000 Subject: [njs] Version 0.6.0. Message-ID: details: https://hg.nginx.org/njs/rev/742ebceef2b5 branches: changeset: 1661:742ebceef2b5 user: Dmitry Volyntsev date: Tue Jun 15 12:05:34 2021 +0000 description: Version 0.6.0. diffstat: CHANGES | 25 +++++++++++++++++++++++++ 1 files changed, 25 insertions(+), 0 deletions(-) diffs (32 lines): diff -r 0ee4118d3cf9 -r 742ebceef2b5 CHANGES --- a/CHANGES Tue Jun 15 11:54:03 2021 +0000 +++ b/CHANGES Tue Jun 15 12:05:34 2021 +0000 @@ -1,3 +1,28 @@ +Changes with njs 0.6.0 15 Jun 2021 + + Core: + + *) Feature: added let and const declaration support. + + *) Feature: added RegExp.prototype[Symbol.split]. + + *) Feature: added sticky flag support for RegExp. + + *) Bugfix: fixed heap-buffer-overflow in + String.prototype.lastIndexOf(). + + *) Bugfix: fixed RegExp.prototype.test() according to the + specification. + + *) Bugfix: fixed String.prototype.split() according to the + specification. + + *) Bugfix: fixed use-of-uninitialized-value while tracking + rejected promises. + + *) Bugfix: fixed njs.dump() for objects with circular + references. + Changes with njs 0.5.3 30 Mar 2021 nginx modules: From xeioex at nginx.com Tue Jun 15 12:54:37 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 15 Jun 2021 12:54:37 +0000 Subject: [njs] Added tag 0.6.0 for changeset 742ebceef2b5 Message-ID: details: https://hg.nginx.org/njs/rev/f6c7e1cf933d branches: changeset: 1662:f6c7e1cf933d user: Dmitry Volyntsev date: Tue Jun 15 12:53:09 2021 +0000 description: Added tag 0.6.0 for changeset 742ebceef2b5 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 742ebceef2b5 -r f6c7e1cf933d .hgtags --- a/.hgtags Tue Jun 15 12:05:34 2021 +0000 +++ b/.hgtags Tue Jun 15 12:53:09 2021 +0000 @@ -42,3 +42,4 @@ 69f07c6151628880bf7d5ac28bd8287ce96d8a36 d355071f55ef4612d89db0ba72e7aaeaa99deef7 0.5.1 e5de01378b1a8ab0a94dd3a8c4c6bb7a235f4b9c 0.5.2 282b9412976ceee31eb12876f1499fe975e6f08c 0.5.3 +742ebceef2b5d15febc093172fe6174e427b26c8 0.6.0 From tracey at archive.org Tue Jun 15 22:49:48 2021 From: tracey at archive.org (Tracey Jaquith) Date: Tue, 15 Jun 2021 15:49:48 -0700 Subject: [PATCH] Add optional "mp4_exact_start" nginx config off/on to show video between keyframes Message-ID: # HG changeset patch # User Tracey Jaquith # Date 1623797180 0 # Tue Jun 15 22:46:20 2021 +0000 # Node ID 1879d49fe0cf739f48287b5a38a83d3a1adab939 # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 Add optional "mp4_exact_start" nginx config off/on to show video between keyframes. archive.org has been using mod_h264_streaming with a similar "exact start" patch from me since 2013. We just moved to nginx mp4 module and are using this patch. The technique is to find the video keyframe just before the desired "start" time, and send that down the wire so video playback can start immediately. Next calculate how many video samples are between the keyframe and desired "start" time and update the STTS atom where those samples move the duration from (typically) 1001 to 1. This way, initial unwanted video frames play at ~1/30,000s -- so visually the video & audio start playing immediately. You can see an example before/after here (nginx binary built with mp4 module + patch): https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30 https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30&exact=1 Tested on linux and macosx. (this is me: https://github.com/traceypooh ) diff -r 5f765427c17a -r 1879d49fe0cf src/http/modules/ngx_http_mp4_module.c --- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300 +++ b/src/http/modules/ngx_http_mp4_module.c Tue Jun 15 22:46:20 2021 +0000 @@ -43,6 +43,7 @@ typedef struct { size_t buffer_size; size_t max_buffer_size; + ngx_flag_t exact_start; } ngx_http_mp4_conf_t; @@ -340,6 +341,13 @@ offsetof(ngx_http_mp4_conf_t, max_buffer_size), NULL }, + { ngx_string("mp4_exact_start"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_mp4_conf_t, exact_start), + NULL }, + ngx_null_command }; @@ -2156,6 +2164,83 @@ static ngx_int_t +ngx_http_mp4_exact_start_video(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak) +{ + uint32_t n, speedup_samples, current_count; + ngx_uint_t sample_keyframe, start_sample_exact; + ngx_mp4_stts_entry_t *entry, *entries_array; + ngx_buf_t *data; + + data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf; + + // Find the keyframe just before the desired start time - so that we can emit an mp4 + // where the first frame is a keyframe. We'll "speed up" the first frames to 1000x + // normal speed (typically), so they won't be noticed. But this way, perceptively, + // playback of the _video_ track can start immediately + // (and not have to wait until the keyframe _after_ the desired starting time frame). + start_sample_exact = trak->start_sample; + for (n = 0; n < trak->sync_samples_entries; n++) { + // each element of array is the sample number of a keyframe + // sync samples starts from 1 -- so subtract 1 + sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4)) - 1; + if (sample_keyframe <= trak->start_sample) { + start_sample_exact = sample_keyframe; + } + if (sample_keyframe >= trak->start_sample) { + break; + } + } + + if (start_sample_exact < trak->start_sample) { + // We're going to prepend an entry with duration=1 for the frames we want to "not see". + // MOST of the time (eg: constant video framerate), + // we're taking a single element entry array and making it two. + speedup_samples = trak->start_sample - start_sample_exact; + + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact trak start_sample move %l to %l (speed up %d samples)\n", + trak->start_sample, start_sample_exact, speedup_samples); + + entries_array = ngx_palloc(mp4->request->pool, + (1 + trak->time_to_sample_entries) * sizeof(ngx_mp4_stts_entry_t)); + if (entries_array == NULL) { + return NGX_ERROR; + } + entry = &(entries_array[1]); + ngx_memcpy(entry, (ngx_mp4_stts_entry_t *)data->pos, + trak->time_to_sample_entries * sizeof(ngx_mp4_stts_entry_t)); + + current_count = ngx_mp4_get_32value(entry->count); + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split in 2 video STTS entry from count:%d", current_count); + + if (current_count <= speedup_samples) { + return NGX_ERROR; + } + + ngx_mp4_set_32value(entry->count, current_count - speedup_samples); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split new[1]: count:%d duration:%d", + ngx_mp4_get_32value(entry->count), + ngx_mp4_get_32value(entry->duration)); + entry--; + ngx_mp4_set_32value(entry->count, speedup_samples); + ngx_mp4_set_32value(entry->duration, 1); + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split new[0]: count:%d duration:1", + ngx_mp4_get_32value(entry->count)); + + data->pos = (u_char *) entry; + trak->time_to_sample_entries++; + trak->start_sample = start_sample_exact; + data->last = (u_char *) (entry + trak->time_to_sample_entries); + } + + return NGX_OK; +} + + +static ngx_int_t ngx_http_mp4_crop_stts_data(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_uint_t start) { @@ -2164,6 +2249,8 @@ ngx_buf_t *data; ngx_uint_t start_sample, entries, start_sec; ngx_mp4_stts_entry_t *entry, *end; + ngx_http_mp4_conf_t *conf; + if (start) { start_sec = mp4->start; @@ -2238,6 +2325,10 @@ "start_sample:%ui, new count:%uD", trak->start_sample, count - rest); + conf = ngx_http_get_module_loc_conf(mp4->request, ngx_http_mp4_module); + if (conf->exact_start) { + ngx_http_mp4_exact_start_video(mp4, trak); + } } else { ngx_mp4_set_32value(entry->count, rest); data->last = (u_char *) (entry + 1); @@ -3590,6 +3681,7 @@ conf->buffer_size = NGX_CONF_UNSET_SIZE; conf->max_buffer_size = NGX_CONF_UNSET_SIZE; + conf->exact_start = NGX_CONF_UNSET; return conf; } From pluknet at nginx.com Thu Jun 17 09:49:35 2021 From: pluknet at nginx.com (Sergey Kandaurov) Date: Thu, 17 Jun 2021 09:49:35 +0000 Subject: [nginx] gRPC: handling GOAWAY with a higher last stream identifier. Message-ID: details: https://hg.nginx.org/nginx/rev/021416fca094 branches: changeset: 7872:021416fca094 user: Sergey Kandaurov date: Thu Jun 17 11:43:55 2021 +0300 description: gRPC: handling GOAWAY with a higher last stream identifier. Previously, once received from upstream, it couldn't limit opening additional streams in a cached keepalive connection. diffstat: src/http/modules/ngx_http_grpc_module.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diffs (61 lines): diff -r 5f765427c17a -r 021416fca094 src/http/modules/ngx_http_grpc_module.c --- a/src/http/modules/ngx_http_grpc_module.c Tue Jun 01 17:37:51 2021 +0300 +++ b/src/http/modules/ngx_http_grpc_module.c Thu Jun 17 11:43:55 2021 +0300 @@ -121,6 +121,7 @@ typedef struct { unsigned done:1; unsigned status:1; unsigned rst:1; + unsigned goaway:1; ngx_http_request_t *request; @@ -1210,6 +1211,7 @@ ngx_http_grpc_reinit_request(ngx_http_re ctx->done = 0; ctx->status = 0; ctx->rst = 0; + ctx->goaway = 0; ctx->connection = NULL; return NGX_OK; @@ -1565,6 +1567,7 @@ ngx_http_grpc_body_output_filter(void *d && ctx->out == NULL && ctx->output_closed && !ctx->output_blocked + && !ctx->goaway && ctx->state == ngx_http_grpc_st_start) { u->keepalive = 1; @@ -1714,6 +1717,8 @@ ngx_http_grpc_process_header(ngx_http_re return NGX_HTTP_UPSTREAM_INVALID_HEADER; } + ctx->goaway = 1; + continue; } @@ -1907,6 +1912,7 @@ ngx_http_grpc_process_header(ngx_http_re && ctx->out == NULL && ctx->output_closed && !ctx->output_blocked + && !ctx->goaway && b->last == b->pos) { u->keepalive = 1; @@ -2035,6 +2041,7 @@ ngx_http_grpc_filter(void *data, ssize_t if (ctx->in == NULL && ctx->output_closed && !ctx->output_blocked + && !ctx->goaway && ctx->state == ngx_http_grpc_st_start) { u->keepalive = 1; @@ -2204,6 +2211,8 @@ ngx_http_grpc_filter(void *data, ssize_t return NGX_ERROR; } + ctx->goaway = 1; + continue; } From pluknet at nginx.com Thu Jun 17 09:49:38 2021 From: pluknet at nginx.com (Sergey Kandaurov) Date: Thu, 17 Jun 2021 09:49:38 +0000 Subject: [nginx] gRPC: RST_STREAM(NO_ERROR) handling micro-optimization. Message-ID: details: https://hg.nginx.org/nginx/rev/0302a4f0b6c4 branches: changeset: 7873:0302a4f0b6c4 user: Sergey Kandaurov date: Thu Jun 17 11:44:06 2021 +0300 description: gRPC: RST_STREAM(NO_ERROR) handling micro-optimization. After 2096b21fcd10, a single RST_STREAM(NO_ERROR) may not result in an error. This change removes several unnecessary ctx->type checks for such a case. diffstat: src/http/modules/ngx_http_grpc_module.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (21 lines): diff -r 021416fca094 -r 0302a4f0b6c4 src/http/modules/ngx_http_grpc_module.c --- a/src/http/modules/ngx_http_grpc_module.c Thu Jun 17 11:43:55 2021 +0300 +++ b/src/http/modules/ngx_http_grpc_module.c Thu Jun 17 11:44:06 2021 +0300 @@ -2177,6 +2177,8 @@ ngx_http_grpc_filter(void *data, ssize_t } ctx->rst = 1; + + continue; } if (ctx->type == NGX_HTTP_V2_GOAWAY_FRAME) { @@ -3484,6 +3486,8 @@ ngx_http_grpc_parse_rst_stream(ngx_http_ return NGX_AGAIN; } + ctx->state = ngx_http_grpc_st_start; + return NGX_OK; } From jusmaki at gmail.com Thu Jun 17 19:44:47 2021 From: jusmaki at gmail.com (Jussi Maki) Date: Thu, 17 Jun 2021 22:44:47 +0300 Subject: Support dynamic client_max_body_size and new response_max_body_size Message-ID: Hi, have there been plans on adding support for dynamic client_max_body_size, based on nginx-variable so it could be set for example in Lua-code per request? Also another nice feature to support dynamic size limitations would be a response size body limit like response_max_body_size with support for dynamic limits using a variable? If this is not planned for any near future I could try to see how challenging this kind of support would be to add. I guess for the client_max_body_size variable resolution would be reasonably simple following example from some other existing dynamic variables. However, adding support for response_max_body_size would require a bit more changes and also in the case of chunked-encoding a good question would be what should be done if the content-size limit exceeds during the response, should the connection be closed or what. Or perhaps this kind of feature would only be supported for responses with Content-Length header? Alternatively these limitations could be done by perhaps using some Lua code and ngx.location.capture but I think it would be cleaner to have a dynamic control variable for the limits. br, Jussi -------------- next part -------------- An HTML attachment was scrubbed... URL: From mdounin at mdounin.ru Fri Jun 18 02:11:58 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Fri, 18 Jun 2021 02:11:58 +0000 Subject: [nginx] Fixed format strings for ngx_win32_version. Message-ID: details: https://hg.nginx.org/nginx/rev/d1079d6b2f19 branches: changeset: 7874:d1079d6b2f19 user: Maxim Dounin date: Fri Jun 18 04:00:21 2021 +0300 description: Fixed format strings for ngx_win32_version. diffstat: src/os/win32/ngx_win32_init.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (30 lines): diff -r 0302a4f0b6c4 -r d1079d6b2f19 src/os/win32/ngx_win32_init.c --- a/src/os/win32/ngx_win32_init.c Thu Jun 17 11:44:06 2021 +0300 +++ b/src/os/win32/ngx_win32_init.c Fri Jun 18 04:00:21 2021 +0300 @@ -295,7 +295,7 @@ ngx_os_status(ngx_log_t *log) osviex_stub = (ngx_osviex_stub_t *) &osvi.wServicePackMinor; ngx_log_error(NGX_LOG_INFO, log, 0, - "OS: %ud build:%ud, \"%s\", suite:%Xd, type:%ud", + "OS: %ui build:%ud, \"%s\", suite:%Xd, type:%ud", ngx_win32_version, osvi.dwBuildNumber, osvi.szCSDVersion, osviex_stub->wSuiteMask, osviex_stub->wProductType); @@ -305,7 +305,7 @@ ngx_os_status(ngx_log_t *log) /* Win9x build */ ngx_log_error(NGX_LOG_INFO, log, 0, - "OS: %u build:%ud.%ud.%ud, \"%s\"", + "OS: %ui build:%ud.%ud.%ud, \"%s\"", ngx_win32_version, osvi.dwBuildNumber >> 24, (osvi.dwBuildNumber >> 16) & 0xff, @@ -321,7 +321,7 @@ ngx_os_status(ngx_log_t *log) * and we do not support VER_PLATFORM_WIN32s at all */ - ngx_log_error(NGX_LOG_INFO, log, 0, "OS: %ud build:%ud, \"%s\"", + ngx_log_error(NGX_LOG_INFO, log, 0, "OS: %ui build:%ud, \"%s\"", ngx_win32_version, osvi.dwBuildNumber, osvi.szCSDVersion); } From xeioex at nginx.com Fri Jun 18 15:07:37 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 18 Jun 2021 15:07:37 +0000 Subject: [njs] Version bump. Message-ID: details: https://hg.nginx.org/njs/rev/8d4bf6ec4002 branches: changeset: 1663:8d4bf6ec4002 user: Dmitry Volyntsev date: Fri Jun 18 15:00:32 2021 +0000 description: Version bump. diffstat: src/njs.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r f6c7e1cf933d -r 8d4bf6ec4002 src/njs.h --- a/src/njs.h Tue Jun 15 12:53:09 2021 +0000 +++ b/src/njs.h Fri Jun 18 15:00:32 2021 +0000 @@ -11,7 +11,7 @@ #include -#define NJS_VERSION "0.6.0" +#define NJS_VERSION "0.6.1" #include /* STDOUT_FILENO, STDERR_FILENO */ From xeioex at nginx.com Fri Jun 18 15:07:38 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 18 Jun 2021 15:07:38 +0000 Subject: [njs] Printing AST before trying to generate code. Message-ID: details: https://hg.nginx.org/njs/rev/d66e00e55b96 branches: changeset: 1664:d66e00e55b96 user: Dmitry Volyntsev date: Fri Jun 18 15:01:12 2021 +0000 description: Printing AST before trying to generate code. diffstat: src/njs_vm.c | 34 +++++++++++++++++----------------- 1 files changed, 17 insertions(+), 17 deletions(-) diffs (51 lines): diff -r 8d4bf6ec4002 -r d66e00e55b96 src/njs_vm.c --- a/src/njs_vm.c Fri Jun 18 15:00:32 2021 +0000 +++ b/src/njs_vm.c Fri Jun 18 15:01:12 2021 +0000 @@ -150,6 +150,23 @@ njs_vm_compile(njs_vm_t *vm, u_char **st return NJS_ERROR; } + if (njs_slow_path(vm->options.ast)) { + njs_chb_init(&chain, vm->mem_pool); + ret = njs_parser_serialize_ast(parser.node, &chain); + if (njs_slow_path(ret == NJS_ERROR)) { + return ret; + } + + if (njs_slow_path(njs_chb_join(&chain, &ast) != NJS_OK)) { + return NJS_ERROR; + } + + njs_print(ast.start, ast.length); + + njs_chb_destroy(&chain); + njs_mp_free(vm->mem_pool, ast.start); + } + *start = lexer.start; scope = parser.scope; @@ -209,23 +226,6 @@ njs_vm_compile(njs_vm_t *vm, u_char **st njs_disassembler(vm); } - if (njs_slow_path(vm->options.ast)) { - njs_chb_init(&chain, vm->mem_pool); - ret = njs_parser_serialize_ast(parser.node, &chain); - if (njs_slow_path(ret == NJS_ERROR)) { - return ret; - } - - if (njs_slow_path(njs_chb_join(&chain, &ast) != NJS_OK)) { - return NJS_ERROR; - } - - njs_print(ast.start, ast.length); - - njs_chb_destroy(&chain); - njs_mp_free(vm->mem_pool, ast.start); - } - return NJS_OK; } From xeioex at nginx.com Fri Jun 18 15:07:40 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 18 Jun 2021 15:07:40 +0000 Subject: [njs] Fixed printing AST with recently added tokens. Message-ID: details: https://hg.nginx.org/njs/rev/9aed07704f30 branches: changeset: 1665:9aed07704f30 user: Dmitry Volyntsev date: Fri Jun 18 15:01:13 2021 +0000 description: Fixed printing AST with recently added tokens. diffstat: src/njs_parser.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (27 lines): diff -r d66e00e55b96 -r 9aed07704f30 src/njs_parser.c --- a/src/njs_parser.c Fri Jun 18 15:01:12 2021 +0000 +++ b/src/njs_parser.c Fri Jun 18 15:01:13 2021 +0000 @@ -8848,6 +8848,7 @@ njs_parser_serialize_node(njs_chb_t *cha njs_token_serialize(NJS_TOKEN_PROTO_INIT); njs_token_serialize(NJS_TOKEN_FUNCTION); + njs_token_serialize(NJS_TOKEN_FUNCTION_DECLARATION); njs_token_serialize(NJS_TOKEN_FUNCTION_EXPRESSION); njs_token_serialize(NJS_TOKEN_FUNCTION_CALL); njs_token_serialize(NJS_TOKEN_METHOD_CALL); @@ -8858,6 +8859,7 @@ njs_parser_serialize_node(njs_chb_t *cha njs_token_serialize(NJS_TOKEN_BLOCK); njs_token_serialize(NJS_TOKEN_VAR); njs_token_serialize(NJS_TOKEN_LET); + njs_token_serialize(NJS_TOKEN_CONST); njs_token_serialize(NJS_TOKEN_IF); njs_token_serialize(NJS_TOKEN_ELSE); njs_token_serialize(NJS_TOKEN_BRANCHING); @@ -8887,7 +8889,6 @@ njs_parser_serialize_node(njs_chb_t *cha njs_token_serialize(NJS_TOKEN_META); njs_token_serialize(NJS_TOKEN_ASYNC); njs_token_serialize(NJS_TOKEN_AWAIT); - njs_token_serialize(NJS_TOKEN_CONST); njs_token_serialize(NJS_TOKEN_DEBUGGER); njs_token_serialize(NJS_TOKEN_ENUM); From xeioex at nginx.com Fri Jun 18 15:07:42 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 18 Jun 2021 15:07:42 +0000 Subject: [njs] Introduced "debugger" token support. Message-ID: details: https://hg.nginx.org/njs/rev/7717b6523cd4 branches: changeset: 1666:7717b6523cd4 user: Dmitry Volyntsev date: Fri Jun 18 15:01:48 2021 +0000 description: Introduced "debugger" token support. diffstat: src/njs_disassembler.c | 3 +++ src/njs_generator.c | 23 +++++++++++++++++++++++ src/njs_parser.c | 19 +++++++++++++++++-- src/njs_scope.c | 7 +++++++ src/njs_scope.h | 1 + src/njs_vmcode.c | 29 +++++++++++++++++++++++++++++ src/njs_vmcode.h | 7 +++++++ src/test/njs_unit_test.c | 17 +++++++++++++++++ 8 files changed, 104 insertions(+), 2 deletions(-) diffs (235 lines): diff -r 9aed07704f30 -r 7717b6523cd4 src/njs_disassembler.c --- a/src/njs_disassembler.c Fri Jun 18 15:01:13 2021 +0000 +++ b/src/njs_disassembler.c Fri Jun 18 15:01:48 2021 +0000 @@ -153,6 +153,9 @@ static njs_code_name_t code_names[] = { { NJS_VMCODE_ASSIGNMENT_ERROR, sizeof(njs_vmcode_variable_t), njs_str("ASSIGNMENT ERROR") }, + + { NJS_VMCODE_DEBUGGER, sizeof(njs_vmcode_debugger_t), + njs_str("DEBUGGER ") }, }; diff -r 9aed07704f30 -r 7717b6523cd4 src/njs_generator.c --- a/src/njs_generator.c Fri Jun 18 15:01:13 2021 +0000 +++ b/src/njs_generator.c Fri Jun 18 15:01:48 2021 +0000 @@ -111,6 +111,8 @@ static njs_int_t njs_generate_continue_s njs_generator_t *generator, njs_parser_node_t *node); static njs_int_t njs_generate_break_statement(njs_vm_t *vm, njs_generator_t *generator, njs_parser_node_t *node); +static njs_int_t njs_generate_debugger_statement(njs_vm_t *vm, + njs_generator_t *generator, njs_parser_node_t *node); static njs_int_t njs_generate_statement(njs_vm_t *vm, njs_generator_t *generator, njs_parser_node_t *node); static njs_int_t njs_generate_block_statement(njs_vm_t *vm, @@ -317,6 +319,9 @@ njs_generate(njs_vm_t *vm, njs_generator case NJS_TOKEN_BREAK: return njs_generate_break_statement(vm, generator, node); + case NJS_TOKEN_DEBUGGER: + return njs_generate_debugger_statement(vm, generator, node); + case NJS_TOKEN_STATEMENT: return njs_generate_statement(vm, generator, node); @@ -1820,6 +1825,24 @@ syntax_error: static njs_int_t +njs_generate_debugger_statement(njs_vm_t *vm, njs_generator_t *generator, + njs_parser_node_t *node) +{ + njs_vmcode_debugger_t *debugger; + + njs_generate_code(generator, njs_vmcode_debugger_t, debugger, + NJS_VMCODE_DEBUGGER, 0, node); + + debugger->retval = njs_generate_dest_index(vm, generator, node); + if (njs_slow_path(debugger->retval == NJS_INDEX_ERROR)) { + return debugger->retval; + } + + return NJS_OK; +} + + +static njs_int_t njs_generate_statement(njs_vm_t *vm, njs_generator_t *generator, njs_parser_node_t *node) { diff -r 9aed07704f30 -r 7717b6523cd4 src/njs_parser.c --- a/src/njs_parser.c Fri Jun 18 15:01:13 2021 +0000 +++ b/src/njs_parser.c Fri Jun 18 15:01:48 2021 +0000 @@ -6545,7 +6545,22 @@ static njs_int_t njs_parser_debugger_statement(njs_parser_t *parser, njs_lexer_token_t *token, njs_queue_link_t *current) { - return njs_parser_not_supported(parser, token); + parser->node = njs_parser_node_new(parser, NJS_TOKEN_DEBUGGER); + if (parser->node == NULL) { + return NJS_ERROR; + } + + parser->node->token_line = parser->line; + + if (token->type != NJS_TOKEN_SEMICOLON + && token->type != NJS_TOKEN_END) + { + return njs_parser_failed(parser); + } + + njs_lexer_consume_token(parser->lexer, 1); + + return njs_parser_stack_pop(parser); } @@ -8882,6 +8897,7 @@ njs_parser_serialize_node(njs_chb_t *cha njs_token_serialize(NJS_TOKEN_EVAL); njs_token_serialize(NJS_TOKEN_IMPORT); njs_token_serialize(NJS_TOKEN_EXPORT); + njs_token_serialize(NJS_TOKEN_DEBUGGER); #if 0 @@ -8889,7 +8905,6 @@ njs_parser_serialize_node(njs_chb_t *cha njs_token_serialize(NJS_TOKEN_META); njs_token_serialize(NJS_TOKEN_ASYNC); njs_token_serialize(NJS_TOKEN_AWAIT); - njs_token_serialize(NJS_TOKEN_DEBUGGER); njs_token_serialize(NJS_TOKEN_ENUM); njs_token_serialize(NJS_TOKEN_CLASS); diff -r 9aed07704f30 -r 7717b6523cd4 src/njs_scope.c --- a/src/njs_scope.c Fri Jun 18 15:01:13 2021 +0000 +++ b/src/njs_scope.c Fri Jun 18 15:01:48 2021 +0000 @@ -111,6 +111,13 @@ njs_scope_global_index(njs_vm_t *vm, con } +njs_value_t * +njs_scope_value_get(njs_vm_t *vm, njs_index_t index) +{ + return njs_scope_value(vm, index); +} + + static njs_int_t njs_scope_values_hash_test(njs_lvlhsh_query_t *lhq, void *data) { diff -r 9aed07704f30 -r 7717b6523cd4 src/njs_scope.h --- a/src/njs_scope.h Fri Jun 18 15:01:13 2021 +0000 +++ b/src/njs_scope.h Fri Jun 18 15:01:48 2021 +0000 @@ -23,6 +23,7 @@ njs_value_t *njs_scope_create_index_valu njs_value_t **njs_scope_make(njs_vm_t *vm, uint32_t count); njs_index_t njs_scope_global_index(njs_vm_t *vm, const njs_value_t *src, njs_uint_t runtime); +njs_value_t *njs_scope_value_get(njs_vm_t *vm, njs_index_t index); njs_inline njs_index_t diff -r 9aed07704f30 -r 7717b6523cd4 src/njs_vmcode.c --- a/src/njs_vmcode.c Fri Jun 18 15:01:13 2021 +0000 +++ b/src/njs_vmcode.c Fri Jun 18 15:01:48 2021 +0000 @@ -37,6 +37,7 @@ static njs_jump_off_t njs_vmcode_instanc njs_value_t *constructor); static njs_jump_off_t njs_vmcode_typeof(njs_vm_t *vm, njs_value_t *value, njs_value_t *invld); +static njs_jump_off_t njs_vmcode_debugger(njs_vm_t *vm); static njs_jump_off_t njs_vmcode_return(njs_vm_t *vm, njs_value_t *invld, njs_value_t *retval); @@ -603,6 +604,10 @@ next: ret = sizeof(njs_vmcode_2addr_t); break; + case NJS_VMCODE_DEBUGGER: + ret = njs_vmcode_debugger(vm); + break; + default: njs_internal_error(vm, "%d has retval", op); goto error; @@ -1517,6 +1522,30 @@ njs_vmcode_typeof(njs_vm_t *vm, njs_valu static njs_jump_off_t +njs_vmcode_debugger(njs_vm_t *vm) +{ + /* + * HOW TO DEBUG JS CODE: + * 1) put debugger instruction when certain condition is met + * in the JS code: + * function test() { + * if (<>) debugger; + * } + * 2) set a breakpoint to njs_vmcode_debugger() in gdb. + * + * To see the values of certain indices: + * 1) use njs -d test.js to dump the byte code + * 2) find an appropriate index around DEBUGGER instruction + * 3) in gdb: p *njs_scope_value_get(vm, ) + **/ + + njs_set_undefined(&vm->retval); + + return sizeof(njs_vmcode_debugger_t); +} + + +static njs_jump_off_t njs_string_concat(njs_vm_t *vm, njs_value_t *val1, njs_value_t *val2) { u_char *start; diff -r 9aed07704f30 -r 7717b6523cd4 src/njs_vmcode.h --- a/src/njs_vmcode.h Fri Jun 18 15:01:13 2021 +0000 +++ b/src/njs_vmcode.h Fri Jun 18 15:01:48 2021 +0000 @@ -126,6 +126,7 @@ enum { NJS_VMCODE_TYPEOF, NJS_VMCODE_VOID, NJS_VMCODE_DELETE, + NJS_VMCODE_DEBUGGER, NJS_VMCODE_NOP = 255 }; @@ -421,6 +422,12 @@ typedef struct { } njs_vmcode_variable_t; +typedef struct { + njs_vmcode_t code; + njs_index_t retval; +} njs_vmcode_debugger_t; + + njs_int_t njs_vmcode_interpreter(njs_vm_t *vm, u_char *pc); njs_object_t *njs_function_new_object(njs_vm_t *vm, njs_value_t *constructor); diff -r 9aed07704f30 -r 7717b6523cd4 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Fri Jun 18 15:01:13 2021 +0000 +++ b/src/test/njs_unit_test.c Fri Jun 18 15:01:48 2021 +0000 @@ -16562,6 +16562,23 @@ static njs_unit_test_t njs_test[] = { njs_str("parseFloat('-5.7e+abc')"), njs_str("-5.7") }, + /* debugger. */ + + { njs_str("debugger"), + njs_str("undefined") }, + + { njs_str("debugger;"), + njs_str("undefined") }, + + { njs_str("while (false) debugger;"), + njs_str("undefined") }, + + { njs_str("1 + debugger"), + njs_str("SyntaxError: Unexpected token \"debugger\" in 1") }, + + { njs_str("debugger + 1"), + njs_str("SyntaxError: Unexpected token \"+\" in 1") }, + /* Top-level objects. */ { njs_str("var global = this;" From vl at nginx.com Mon Jun 21 14:59:02 2021 From: vl at nginx.com (Vladimir Homutov) Date: Mon, 21 Jun 2021 14:59:02 +0000 Subject: [nginx] Core: added the ngx_rbtree_data() macro. Message-ID: details: https://hg.nginx.org/nginx/rev/0c5e84096d99 branches: changeset: 7875:0c5e84096d99 user: Vladimir Homutov date: Mon Jun 21 09:42:43 2021 +0300 description: Core: added the ngx_rbtree_data() macro. diffstat: src/core/ngx_rbtree.h | 3 +++ src/core/ngx_resolver.c | 4 +--- src/event/ngx_event_timer.c | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diffs (48 lines): diff -r d1079d6b2f19 -r 0c5e84096d99 src/core/ngx_rbtree.h --- a/src/core/ngx_rbtree.h Fri Jun 18 04:00:21 2021 +0300 +++ b/src/core/ngx_rbtree.h Mon Jun 21 09:42:43 2021 +0300 @@ -47,6 +47,9 @@ struct ngx_rbtree_s { (tree)->sentinel = s; \ (tree)->insert = i +#define ngx_rbtree_data(node, type, link) \ + (type *) ((u_char *) (node) - offsetof(type, link)) + void ngx_rbtree_insert(ngx_rbtree_t *tree, ngx_rbtree_node_t *node); void ngx_rbtree_delete(ngx_rbtree_t *tree, ngx_rbtree_node_t *node); diff -r d1079d6b2f19 -r 0c5e84096d99 src/core/ngx_resolver.c --- a/src/core/ngx_resolver.c Fri Jun 18 04:00:21 2021 +0300 +++ b/src/core/ngx_resolver.c Mon Jun 21 09:42:43 2021 +0300 @@ -51,9 +51,7 @@ typedef struct { } ngx_resolver_an_t; -#define ngx_resolver_node(n) \ - (ngx_resolver_node_t *) \ - ((u_char *) (n) - offsetof(ngx_resolver_node_t, node)) +#define ngx_resolver_node(n) ngx_rbtree_data(n, ngx_resolver_node_t, node) static ngx_int_t ngx_udp_connect(ngx_resolver_connection_t *rec); diff -r d1079d6b2f19 -r 0c5e84096d99 src/event/ngx_event_timer.c --- a/src/event/ngx_event_timer.c Fri Jun 18 04:00:21 2021 +0300 +++ b/src/event/ngx_event_timer.c Mon Jun 21 09:42:43 2021 +0300 @@ -73,7 +73,7 @@ ngx_event_expire_timers(void) return; } - ev = (ngx_event_t *) ((char *) node - offsetof(ngx_event_t, timer)); + ev = ngx_rbtree_data(node, ngx_event_t, timer); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, "event timer del: %d: %M", @@ -113,7 +113,7 @@ ngx_event_no_timers_left(void) node; node = ngx_rbtree_next(&ngx_event_timer_rbtree, node)) { - ev = (ngx_event_t *) ((char *) node - offsetof(ngx_event_t, timer)); + ev = ngx_rbtree_data(node, ngx_event_t, timer); if (!ev->cancelable) { return NGX_AGAIN; From xeioex at nginx.com Fri Jun 25 17:33:15 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Fri, 25 Jun 2021 17:33:15 +0000 Subject: [njs] Fixed RegExpBuiltinExec() with UTF-8 only regexps. Message-ID: details: https://hg.nginx.org/njs/rev/f10d5c38f098 branches: changeset: 1667:f10d5c38f098 user: Dmitry Volyntsev date: Fri Jun 25 17:00:12 2021 +0000 description: Fixed RegExpBuiltinExec() with UTF-8 only regexps. The original issue was introduced in f9082cd59ba6 (0.4.2) while adding RegExpBuiltinExec(), but after de64420d0f2b (0.6.0) it started to affect RegExp.prototype.test() as it was rewritten according to spec. diffstat: src/njs_regexp.c | 24 ++++++++++++++---------- src/test/njs_unit_test.c | 13 +++++++++++++ 2 files changed, 27 insertions(+), 10 deletions(-) diffs (123 lines): diff -r 7717b6523cd4 -r f10d5c38f098 src/njs_regexp.c --- a/src/njs_regexp.c Fri Jun 18 15:01:48 2021 +0000 +++ b/src/njs_regexp.c Fri Jun 25 17:00:12 2021 +0000 @@ -26,8 +26,7 @@ static u_char *njs_regexp_compile_trace_ static u_char *njs_regexp_match_trace_handler(njs_trace_t *trace, njs_trace_data_t *td, u_char *start); static njs_array_t *njs_regexp_exec_result(njs_vm_t *vm, njs_value_t *r, - njs_regexp_utf8_t type, njs_string_prop_t *string, - njs_regex_match_data_t *data); + njs_utf8_t utf8, njs_string_prop_t *string, njs_regex_match_data_t *data); static njs_int_t njs_regexp_string_create(njs_vm_t *vm, njs_value_t *value, u_char *start, uint32_t size, int32_t length); @@ -946,6 +945,7 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj size_t length, offset; int64_t last_index; njs_int_t ret; + njs_utf8_t utf8; njs_value_t value; njs_array_t *result; njs_regexp_t *regexp; @@ -979,11 +979,15 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj goto not_found; } + utf8 = NJS_STRING_BYTE; type = NJS_REGEXP_BYTE; - if (length != string.size) { - /* UTF-8 string. */ + if (string.length != 0) { type = NJS_REGEXP_UTF8; + + if (string.length != string.size) { + utf8 = NJS_STRING_UTF8; + } } pattern = regexp->pattern; @@ -998,7 +1002,7 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj return NJS_ERROR; } - if (type != NJS_REGEXP_UTF8) { + if (utf8 != NJS_STRING_UTF8) { offset = last_index; } else { @@ -1010,7 +1014,7 @@ njs_regexp_builtin_exec(njs_vm_t *vm, nj ret = njs_regexp_match(vm, &pattern->regex[type], string.start, offset, string.size, match_data); if (ret >= 0) { - result = njs_regexp_exec_result(vm, r, type, &string, match_data); + result = njs_regexp_exec_result(vm, r, utf8, &string, match_data); if (njs_slow_path(result == NULL)) { return NJS_ERROR; } @@ -1043,7 +1047,7 @@ not_found: static njs_array_t * -njs_regexp_exec_result(njs_vm_t *vm, njs_value_t *r, njs_regexp_utf8_t type, +njs_regexp_exec_result(njs_vm_t *vm, njs_value_t *r, njs_utf8_t utf8, njs_string_prop_t *string, njs_regex_match_data_t *match_data) { int *captures; @@ -1081,7 +1085,7 @@ njs_regexp_exec_result(njs_vm_t *vm, njs start = &string->start[captures[n]]; size = captures[n + 1] - captures[n]; - if (type == NJS_REGEXP_UTF8) { + if (utf8 == NJS_STRING_UTF8) { length = njs_max(njs_utf8_length(start, size), 0); } else { @@ -1105,7 +1109,7 @@ njs_regexp_exec_result(njs_vm_t *vm, njs goto fail; } - if (type == NJS_REGEXP_UTF8) { + if (utf8 == NJS_STRING_UTF8) { index = njs_string_index(string, captures[0]); } else { @@ -1115,7 +1119,7 @@ njs_regexp_exec_result(njs_vm_t *vm, njs njs_set_number(&prop->value, index); if (pattern->global || pattern->sticky) { - if (type == NJS_REGEXP_UTF8) { + if (utf8 == NJS_STRING_UTF8) { index = njs_string_index(string, captures[1]); } else { diff -r 7717b6523cd4 -r f10d5c38f098 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Fri Jun 18 15:01:48 2021 +0000 +++ b/src/test/njs_unit_test.c Fri Jun 25 17:00:12 2021 +0000 @@ -10763,6 +10763,12 @@ static njs_unit_test_t njs_test[] = { njs_str("/?/.test('\\u00CE\\u00B1'.toBytes())"), njs_str("true") }, + { njs_str("/[A-Za-z]/.test('S')"), + njs_str("true") }, + + { njs_str("/[A-Za-z]/.test('?')"), + njs_str("false") }, + { njs_str("var r = /abc/y; r.test('abc'); r.lastIndex"), njs_str("3") }, @@ -21004,6 +21010,13 @@ static njs_unit_test_t njs_regexp_test[ { njs_str("RegExp('[\0]').test('\0')"), njs_str("true") }, + + { njs_str("/[A-Za-z\\u00F8-\\u02FF]/.test('S')"), + njs_str("true") }, + + { njs_str("/[A-Za-z\\u00F8-\\u02FF]/.test('?')"), + njs_str("true") }, + }; From arut at nginx.com Mon Jun 28 09:53:20 2021 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 28 Jun 2021 12:53:20 +0300 Subject: [PATCH] Add optional "mp4_exact_start" nginx config off/on to show video between keyframes In-Reply-To: References: Message-ID: <20210628095320.px3ggmmoyjalyv5m@Romans-MacBook-Pro.local> Hi Tracey, On Tue, Jun 15, 2021 at 03:49:48PM -0700, Tracey Jaquith wrote: > # HG changeset patch > # User Tracey Jaquith > # Date 1623797180 0 > # Tue Jun 15 22:46:20 2021 +0000 > # Node ID 1879d49fe0cf739f48287b5a38a83d3a1adab939 > # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 > Add optional "mp4_exact_start" nginx config off/on to show video between keyframes. I've been thinking about a better name for this, but came up with nothing so far. I feel like this name does not give the right clue to the user. Moreover, when this feature is on, the start is not quite "exact", but shifted a few milliseconds into the past. > archive.org has been using mod_h264_streaming with a similar "exact start" patch from me since 2013. > We just moved to nginx mp4 module and are using this patch. > The technique is to find the video keyframe just before the desired "start" time, and send > that down the wire so video playback can start immediately. > Next calculate how many video samples are between the keyframe and desired "start" time > and update the STTS atom where those samples move the duration from (typically) 1001 to 1. > This way, initial unwanted video frames play at ~1/30,000s -- so visually the > video & audio start playing immediately. > > You can see an example before/after here (nginx binary built with mp4 module + patch): > > https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30 > https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30&exact=1 > > Tested on linux and macosx. > > (this is me: https://github.com/traceypooh ) We have a few rules about patches and commit messages like 67-character limit for the first line etc: http://nginx.org/en/docs/contributing_changes.html > diff -r 5f765427c17a -r 1879d49fe0cf src/http/modules/ngx_http_mp4_module.c > --- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300 > +++ b/src/http/modules/ngx_http_mp4_module.c Tue Jun 15 22:46:20 2021 +0000 > @@ -43,6 +43,7 @@ > typedef struct { > size_t buffer_size; > size_t max_buffer_size; > + ngx_flag_t exact_start; > } ngx_http_mp4_conf_t; > > > @@ -340,6 +341,13 @@ > offsetof(ngx_http_mp4_conf_t, max_buffer_size), > NULL }, > > + { ngx_string("mp4_exact_start"), > + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, NGX_CONF_TAKE1 -> NGX_CONF_FLAG > + ngx_conf_set_flag_slot, > + NGX_HTTP_LOC_CONF_OFFSET, > + offsetof(ngx_http_mp4_conf_t, exact_start), > + NULL }, > + > ngx_null_command > }; > > @@ -2156,6 +2164,83 @@ > > > static ngx_int_t > +ngx_http_mp4_exact_start_video(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak) > +{ > + uint32_t n, speedup_samples, current_count; > + ngx_uint_t sample_keyframe, start_sample_exact; > + ngx_mp4_stts_entry_t *entry, *entries_array; > + ngx_buf_t *data; > + > + data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf; > + > + // Find the keyframe just before the desired start time - so that we can emit an mp4 > + // where the first frame is a keyframe. We'll "speed up" the first frames to 1000x > + // normal speed (typically), so they won't be noticed. But this way, perceptively, > + // playback of the _video_ track can start immediately > + // (and not have to wait until the keyframe _after_ the desired starting time frame). > + start_sample_exact = trak->start_sample; > + for (n = 0; n < trak->sync_samples_entries; n++) { > + // each element of array is the sample number of a keyframe > + // sync samples starts from 1 -- so subtract 1 > + sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4)) - 1; This can be simplified by introducing entry/end variables like we usually do. Also, we don't access trak->stss_data_buf directly, but prefer trak->out[NGX_HTTP_MP4_STSS_ATOM].buf. ngx_http_mp4_crop_stss_data() provides an example of iterating over stss atom. > + if (sample_keyframe <= trak->start_sample) { > + start_sample_exact = sample_keyframe; > + } > + if (sample_keyframe >= trak->start_sample) { > + break; > + } > + } > + > + if (start_sample_exact < trak->start_sample) { > + // We're going to prepend an entry with duration=1 for the frames we want to "not see". > + // MOST of the time (eg: constant video framerate), > + // we're taking a single element entry array and making it two. > + speedup_samples = trak->start_sample - start_sample_exact; > + > + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, > + "exact trak start_sample move %l to %l (speed up %d samples)\n", > + trak->start_sample, start_sample_exact, speedup_samples); > + > + entries_array = ngx_palloc(mp4->request->pool, > + (1 + trak->time_to_sample_entries) * sizeof(ngx_mp4_stts_entry_t)); > + if (entries_array == NULL) { > + return NGX_ERROR; > + } > + entry = &(entries_array[1]); > + ngx_memcpy(entry, (ngx_mp4_stts_entry_t *)data->pos, > + trak->time_to_sample_entries * sizeof(ngx_mp4_stts_entry_t)); This reallocation can be avoided. Look at NGX_HTTP_MP4_STSC_START buffer as an example of that. A new 1-element optional buffer NGX_HTTP_MP4_STTS_START can be introduced right before the stts atom data. > + current_count = ngx_mp4_get_32value(entry->count); > + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, > + "exact split in 2 video STTS entry from count:%d", current_count); > + > + if (current_count <= speedup_samples) { > + return NGX_ERROR; > + } > + > + ngx_mp4_set_32value(entry->count, current_count - speedup_samples); > + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, > + "exact split new[1]: count:%d duration:%d", > + ngx_mp4_get_32value(entry->count), > + ngx_mp4_get_32value(entry->duration)); > + entry--; > + ngx_mp4_set_32value(entry->count, speedup_samples); > + ngx_mp4_set_32value(entry->duration, 1); > + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, > + "exact split new[0]: count:%d duration:1", > + ngx_mp4_get_32value(entry->count)); > + > + data->pos = (u_char *) entry; > + trak->time_to_sample_entries++; > + trak->start_sample = start_sample_exact; > + data->last = (u_char *) (entry + trak->time_to_sample_entries); > + } > + > + return NGX_OK; > +} > + > + > +static ngx_int_t > ngx_http_mp4_crop_stts_data(ngx_http_mp4_file_t *mp4, > ngx_http_mp4_trak_t *trak, ngx_uint_t start) > { > @@ -2164,6 +2249,8 @@ > ngx_buf_t *data; > ngx_uint_t start_sample, entries, start_sec; > ngx_mp4_stts_entry_t *entry, *end; > + ngx_http_mp4_conf_t *conf; > + No need for a new empty line here. > if (start) { > start_sec = mp4->start; > @@ -2238,6 +2325,10 @@ > "start_sample:%ui, new count:%uD", > trak->start_sample, count - rest); > > + conf = ngx_http_get_module_loc_conf(mp4->request, ngx_http_mp4_module); > + if (conf->exact_start) { > + ngx_http_mp4_exact_start_video(mp4, trak); > + } > } else { > ngx_mp4_set_32value(entry->count, rest); > data->last = (u_char *) (entry + 1); > @@ -3590,6 +3681,7 @@ > > conf->buffer_size = NGX_CONF_UNSET_SIZE; > conf->max_buffer_size = NGX_CONF_UNSET_SIZE; > + conf->exact_start = NGX_CONF_UNSET; This is not enough, a merge is needed too. > > return conf; > } > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > http://mailman.nginx.org/mailman/listinfo/nginx-devel I've made a POC patch which incorporates the issues I've mentioned. I didn't test is properly and the directive name is still not perfect. -- Roman Arutyunyan -------------- next part -------------- # HG changeset patch # User Roman Arutyunyan # Date 1623416525 -10800 # Fri Jun 11 16:02:05 2021 +0300 # Node ID 4c7512abda4b335309e3c03d98e8ab680d6dbdd7 # Parent 1ebd78df4ce7262967c5dadce7bac454c4086896 Mp4: mp4_seek_key_frame directive. The directive enables including all frames from start time to the most recent key frame in the result. Those frames get the smallest time intervals between them. Based on a patch by Tracey Jaquith. diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c --- a/src/http/modules/ngx_http_mp4_module.c +++ b/src/http/modules/ngx_http_mp4_module.c @@ -21,21 +21,22 @@ #define NGX_HTTP_MP4_STBL_ATOM 9 #define NGX_HTTP_MP4_STSD_ATOM 10 #define NGX_HTTP_MP4_STTS_ATOM 11 -#define NGX_HTTP_MP4_STTS_DATA 12 -#define NGX_HTTP_MP4_STSS_ATOM 13 -#define NGX_HTTP_MP4_STSS_DATA 14 -#define NGX_HTTP_MP4_CTTS_ATOM 15 -#define NGX_HTTP_MP4_CTTS_DATA 16 -#define NGX_HTTP_MP4_STSC_ATOM 17 -#define NGX_HTTP_MP4_STSC_START 18 -#define NGX_HTTP_MP4_STSC_DATA 19 -#define NGX_HTTP_MP4_STSC_END 20 -#define NGX_HTTP_MP4_STSZ_ATOM 21 -#define NGX_HTTP_MP4_STSZ_DATA 22 -#define NGX_HTTP_MP4_STCO_ATOM 23 -#define NGX_HTTP_MP4_STCO_DATA 24 -#define NGX_HTTP_MP4_CO64_ATOM 25 -#define NGX_HTTP_MP4_CO64_DATA 26 +#define NGX_HTTP_MP4_STTS_START 12 +#define NGX_HTTP_MP4_STTS_DATA 13 +#define NGX_HTTP_MP4_STSS_ATOM 14 +#define NGX_HTTP_MP4_STSS_DATA 15 +#define NGX_HTTP_MP4_CTTS_ATOM 16 +#define NGX_HTTP_MP4_CTTS_DATA 17 +#define NGX_HTTP_MP4_STSC_ATOM 18 +#define NGX_HTTP_MP4_STSC_START 19 +#define NGX_HTTP_MP4_STSC_DATA 20 +#define NGX_HTTP_MP4_STSC_END 21 +#define NGX_HTTP_MP4_STSZ_ATOM 22 +#define NGX_HTTP_MP4_STSZ_DATA 23 +#define NGX_HTTP_MP4_STCO_ATOM 24 +#define NGX_HTTP_MP4_STCO_DATA 25 +#define NGX_HTTP_MP4_CO64_ATOM 26 +#define NGX_HTTP_MP4_CO64_DATA 27 #define NGX_HTTP_MP4_LAST_ATOM NGX_HTTP_MP4_CO64_DATA @@ -43,6 +44,7 @@ typedef struct { size_t buffer_size; size_t max_buffer_size; + ngx_flag_t seek_key_frame; } ngx_http_mp4_conf_t; @@ -54,6 +56,12 @@ typedef struct { typedef struct { + u_char count[4]; + u_char duration[4]; +} ngx_mp4_stts_entry_t; + + +typedef struct { uint32_t timescale; uint32_t time_to_sample_entries; uint32_t sample_to_chunk_entries; @@ -95,6 +103,7 @@ typedef struct { ngx_buf_t stbl_atom_buf; ngx_buf_t stsd_atom_buf; ngx_buf_t stts_atom_buf; + ngx_buf_t stts_start_buf; ngx_buf_t stts_data_buf; ngx_buf_t stss_atom_buf; ngx_buf_t stss_data_buf; @@ -111,6 +120,7 @@ typedef struct { ngx_buf_t co64_atom_buf; ngx_buf_t co64_data_buf; + ngx_mp4_stts_entry_t stts_start_entry; ngx_mp4_stsc_entry_t stsc_start_chunk_entry; ngx_mp4_stsc_entry_t stsc_end_chunk_entry; } ngx_http_mp4_trak_t; @@ -277,6 +287,8 @@ static ngx_int_t ngx_http_mp4_update_stt ngx_http_mp4_trak_t *trak); static ngx_int_t ngx_http_mp4_crop_stts_data(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_uint_t start); +static ngx_int_t ngx_http_mp4_rewind_stts_data(ngx_http_mp4_file_t *mp4, + ngx_http_mp4_trak_t *trak); static ngx_int_t ngx_http_mp4_read_stss_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size); static ngx_int_t ngx_http_mp4_update_stss_atom(ngx_http_mp4_file_t *mp4, @@ -340,6 +352,13 @@ static ngx_command_t ngx_http_mp4_comma offsetof(ngx_http_mp4_conf_t, max_buffer_size), NULL }, + { ngx_string("mp4_seek_key_frame"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_mp4_conf_t, seek_key_frame), + NULL }, + ngx_null_command }; @@ -2040,11 +2059,6 @@ typedef struct { u_char entries[4]; } ngx_mp4_stts_atom_t; -typedef struct { - u_char count[4]; - u_char duration[4]; -} ngx_mp4_stts_entry_t; - static ngx_int_t ngx_http_mp4_read_stts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) @@ -2140,10 +2154,20 @@ ngx_http_mp4_update_stts_atom(ngx_http_m return NGX_ERROR; } + if (ngx_http_mp4_rewind_stts_data(mp4, trak) != NGX_OK) { + return NGX_ERROR; + } + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "time-to-sample entries:%uD", trak->time_to_sample_entries); atom_size = sizeof(ngx_mp4_stts_atom_t) + (data->last - data->pos); + + data = trak->out[NGX_HTTP_MP4_STTS_START].buf; + if (data) { + atom_size += data->last - data->pos; + } + trak->size += atom_size; atom = trak->out[NGX_HTTP_MP4_STTS_ATOM].buf; @@ -2253,6 +2277,69 @@ found: } +static ngx_int_t +ngx_http_mp4_rewind_stts_data(ngx_http_mp4_file_t *mp4, + ngx_http_mp4_trak_t *trak) +{ + uint32_t start_sample, rewind, sample, *entry, *end; + ngx_buf_t *data; + ngx_http_mp4_conf_t *conf; + ngx_mp4_stts_entry_t *rewind_entry; + + conf = ngx_http_get_module_loc_conf(mp4->request, ngx_http_mp4_module); + if (!conf->seek_key_frame) { + return NGX_OK; + } + + data = trak->out[NGX_HTTP_MP4_STSS_DATA].buf; + if (data == NULL) { + return NGX_OK; + } + + entry = (uint32_t *) data->pos; + end = (uint32_t *) data->last; + + /* sync samples starts from 1 */ + start_sample = trak->start_sample + 1; + + rewind = 0; + + while (entry < end) { + sample = ngx_mp4_get_32value(entry); + if (sample > start_sample) { + break; + } + + rewind = start_sample - sample; + entry++; + } + + if (rewind == 0) { + return NGX_OK; + } + + trak->time_to_sample_entries++; + trak->start_sample -= rewind; + + rewind_entry = &trak->stts_start_entry; + ngx_mp4_set_32value(rewind_entry->count, rewind); + ngx_mp4_set_32value(rewind_entry->duration, 1); + + data = &trak->stts_start_buf; + data->temporary = 1; + data->pos = (u_char *) rewind_entry; + data->last = (u_char *) rewind_entry + sizeof(ngx_mp4_stts_entry_t); + + trak->out[NGX_HTTP_MP4_STTS_START].buf = data; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "mp4 rewind samples:%uD, start_sample:%ui", + rewind, trak->start_sample); + + return NGX_OK; +} + + typedef struct { u_char size[4]; u_char name[4]; @@ -3590,6 +3677,7 @@ ngx_http_mp4_create_conf(ngx_conf_t *cf) conf->buffer_size = NGX_CONF_UNSET_SIZE; conf->max_buffer_size = NGX_CONF_UNSET_SIZE; + conf->seek_key_frame = NGX_CONF_UNSET; return conf; } @@ -3604,6 +3692,7 @@ ngx_http_mp4_merge_conf(ngx_conf_t *cf, ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size, 512 * 1024); ngx_conf_merge_size_value(conf->max_buffer_size, prev->max_buffer_size, 10 * 1024 * 1024); + ngx_conf_merge_value(conf->seek_key_frame, prev->seek_key_frame, 0); return NGX_CONF_OK; } From mdounin at mdounin.ru Mon Jun 28 18:36:13 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 28 Jun 2021 18:36:13 +0000 Subject: [nginx] Moved TRACE method rejection to a better place. Message-ID: details: https://hg.nginx.org/nginx/rev/b290610bf812 branches: changeset: 7876:b290610bf812 user: Maxim Dounin date: Mon Jun 28 18:01:00 2021 +0300 description: Moved TRACE method rejection to a better place. Previously, TRACE requests were rejected before parsing Transfer-Encoding. This is not important since keepalive is not enabled at this point anyway, though rejecting such requests after properly parsing other headers is less likely to cause issues in case of further code changes. diffstat: src/http/ngx_http_request.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diffs (31 lines): diff -r 0c5e84096d99 -r b290610bf812 src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c Mon Jun 21 09:42:43 2021 +0300 +++ b/src/http/ngx_http_request.c Mon Jun 28 18:01:00 2021 +0300 @@ -1980,13 +1980,6 @@ ngx_http_process_request_header(ngx_http } } - if (r->method == NGX_HTTP_TRACE) { - ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, - "client sent TRACE method"); - ngx_http_finalize_request(r, NGX_HTTP_NOT_ALLOWED); - return NGX_ERROR; - } - if (r->headers_in.transfer_encoding) { if (r->headers_in.transfer_encoding->value.len == 7 && ngx_strncasecmp(r->headers_in.transfer_encoding->value.data, @@ -2013,6 +2006,13 @@ ngx_http_process_request_header(ngx_http } } + if (r->method == NGX_HTTP_TRACE) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent TRACE method"); + ngx_http_finalize_request(r, NGX_HTTP_NOT_ALLOWED); + return NGX_ERROR; + } + return NGX_OK; } From mdounin at mdounin.ru Mon Jun 28 18:36:16 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 28 Jun 2021 18:36:16 +0000 Subject: [nginx] Added CONNECT method rejection. Message-ID: details: https://hg.nginx.org/nginx/rev/63c66b7cc07c branches: changeset: 7877:63c66b7cc07c user: Maxim Dounin date: Mon Jun 28 18:01:04 2021 +0300 description: Added CONNECT method rejection. No valid CONNECT requests are expected to appear within nginx, since it is not a forward proxy. Further, request line parsing will reject proper CONNECT requests anyway, since we don't allow authority-form of request-target. On the other hand, RFC 7230 specifies separate message length rules for CONNECT which we don't support, so make sure to always reject CONNECTs to avoid potential abuse. diffstat: src/http/ngx_http_parse.c | 5 +++++ src/http/ngx_http_request.c | 7 +++++++ src/http/ngx_http_request.h | 33 +++++++++++++++++---------------- src/http/v2/ngx_http_v2.c | 3 ++- 4 files changed, 31 insertions(+), 17 deletions(-) diffs (88 lines): diff -r b290610bf812 -r 63c66b7cc07c src/http/ngx_http_parse.c --- a/src/http/ngx_http_parse.c Mon Jun 28 18:01:00 2021 +0300 +++ b/src/http/ngx_http_parse.c Mon Jun 28 18:01:04 2021 +0300 @@ -246,6 +246,11 @@ ngx_http_parse_request_line(ngx_http_req r->method = NGX_HTTP_OPTIONS; } + if (ngx_str7_cmp(m, 'C', 'O', 'N', 'N', 'E', 'C', 'T', ' ')) + { + r->method = NGX_HTTP_CONNECT; + } + break; case 8: diff -r b290610bf812 -r 63c66b7cc07c src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c Mon Jun 28 18:01:00 2021 +0300 +++ b/src/http/ngx_http_request.c Mon Jun 28 18:01:04 2021 +0300 @@ -2006,6 +2006,13 @@ ngx_http_process_request_header(ngx_http } } + if (r->method == NGX_HTTP_CONNECT) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent CONNECT method"); + ngx_http_finalize_request(r, NGX_HTTP_NOT_ALLOWED); + return NGX_ERROR; + } + if (r->method == NGX_HTTP_TRACE) { ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, "client sent TRACE method"); diff -r b290610bf812 -r 63c66b7cc07c src/http/ngx_http_request.h --- a/src/http/ngx_http_request.h Mon Jun 28 18:01:00 2021 +0300 +++ b/src/http/ngx_http_request.h Mon Jun 28 18:01:04 2021 +0300 @@ -25,22 +25,23 @@ #define NGX_HTTP_VERSION_11 1001 #define NGX_HTTP_VERSION_20 2000 -#define NGX_HTTP_UNKNOWN 0x0001 -#define NGX_HTTP_GET 0x0002 -#define NGX_HTTP_HEAD 0x0004 -#define NGX_HTTP_POST 0x0008 -#define NGX_HTTP_PUT 0x0010 -#define NGX_HTTP_DELETE 0x0020 -#define NGX_HTTP_MKCOL 0x0040 -#define NGX_HTTP_COPY 0x0080 -#define NGX_HTTP_MOVE 0x0100 -#define NGX_HTTP_OPTIONS 0x0200 -#define NGX_HTTP_PROPFIND 0x0400 -#define NGX_HTTP_PROPPATCH 0x0800 -#define NGX_HTTP_LOCK 0x1000 -#define NGX_HTTP_UNLOCK 0x2000 -#define NGX_HTTP_PATCH 0x4000 -#define NGX_HTTP_TRACE 0x8000 +#define NGX_HTTP_UNKNOWN 0x00000001 +#define NGX_HTTP_GET 0x00000002 +#define NGX_HTTP_HEAD 0x00000004 +#define NGX_HTTP_POST 0x00000008 +#define NGX_HTTP_PUT 0x00000010 +#define NGX_HTTP_DELETE 0x00000020 +#define NGX_HTTP_MKCOL 0x00000040 +#define NGX_HTTP_COPY 0x00000080 +#define NGX_HTTP_MOVE 0x00000100 +#define NGX_HTTP_OPTIONS 0x00000200 +#define NGX_HTTP_PROPFIND 0x00000400 +#define NGX_HTTP_PROPPATCH 0x00000800 +#define NGX_HTTP_LOCK 0x00001000 +#define NGX_HTTP_UNLOCK 0x00002000 +#define NGX_HTTP_PATCH 0x00004000 +#define NGX_HTTP_TRACE 0x00008000 +#define NGX_HTTP_CONNECT 0x00010000 #define NGX_HTTP_CONNECTION_CLOSE 1 #define NGX_HTTP_CONNECTION_KEEP_ALIVE 2 diff -r b290610bf812 -r 63c66b7cc07c src/http/v2/ngx_http_v2.c --- a/src/http/v2/ngx_http_v2.c Mon Jun 28 18:01:00 2021 +0300 +++ b/src/http/v2/ngx_http_v2.c Mon Jun 28 18:01:04 2021 +0300 @@ -3606,7 +3606,8 @@ ngx_http_v2_parse_method(ngx_http_reques { 4, "LOCK", NGX_HTTP_LOCK }, { 6, "UNLOCK", NGX_HTTP_UNLOCK }, { 5, "PATCH", NGX_HTTP_PATCH }, - { 5, "TRACE", NGX_HTTP_TRACE } + { 5, "TRACE", NGX_HTTP_TRACE }, + { 7, "CONNECT", NGX_HTTP_CONNECT } }, *test; if (r->method_name.len) { From mdounin at mdounin.ru Mon Jun 28 18:36:19 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 28 Jun 2021 18:36:19 +0000 Subject: [nginx] Disabled requests with both Content-Length and Transfer-Encoding. Message-ID: details: https://hg.nginx.org/nginx/rev/bea0f9e5c309 branches: changeset: 7878:bea0f9e5c309 user: Maxim Dounin date: Mon Jun 28 18:01:06 2021 +0300 description: Disabled requests with both Content-Length and Transfer-Encoding. HTTP clients are not allowed to generate such requests since Transfer-Encoding introduction in RFC 2068, and they are not expected to appear in practice except in attempts to perform a request smuggling attack. While handling of such requests is strictly defined, the most secure approach seems to reject them. diffstat: src/http/ngx_http_request.c | 11 +++++++++-- 1 files changed, 9 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 63c66b7cc07c -r bea0f9e5c309 src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c Mon Jun 28 18:01:04 2021 +0300 +++ b/src/http/ngx_http_request.c Mon Jun 28 18:01:06 2021 +0300 @@ -1985,8 +1985,15 @@ ngx_http_process_request_header(ngx_http && ngx_strncasecmp(r->headers_in.transfer_encoding->value.data, (u_char *) "chunked", 7) == 0) { - r->headers_in.content_length = NULL; - r->headers_in.content_length_n = -1; + if (r->headers_in.content_length) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent \"Content-Length\" and " + "\"Transfer-Encoding\" headers " + "at the same time"); + ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); + return NGX_ERROR; + } + r->headers_in.chunked = 1; } else { From mdounin at mdounin.ru Mon Jun 28 18:36:22 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 28 Jun 2021 18:36:22 +0000 Subject: [nginx] Core: fixed comment about escaping in arguments. Message-ID: details: https://hg.nginx.org/nginx/rev/b7407334c60d branches: changeset: 7879:b7407334c60d user: Maxim Dounin date: Mon Jun 28 18:01:09 2021 +0300 description: Core: fixed comment about escaping in arguments. After 4954530db2af, the ";" character is escaped by ngx_escape_uri(NGX_ESCAPE_ARGS). diffstat: src/core/ngx_string.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r bea0f9e5c309 -r b7407334c60d src/core/ngx_string.c --- a/src/core/ngx_string.c Mon Jun 28 18:01:06 2021 +0300 +++ b/src/core/ngx_string.c Mon Jun 28 18:01:09 2021 +0300 @@ -1513,7 +1513,7 @@ ngx_escape_uri(u_char *dst, u_char *src, 0xffffffff /* 1111 1111 1111 1111 1111 1111 1111 1111 */ }; - /* " ", "#", "%", "&", "+", "?", %00-%1F, %7F-%FF */ + /* " ", "#", "%", "&", "+", ";", "?", %00-%1F, %7F-%FF */ static uint32_t args[] = { 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ From mdounin at mdounin.ru Mon Jun 28 18:36:25 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 28 Jun 2021 18:36:25 +0000 Subject: [nginx] Core: escaping of chars not allowed in URIs per RFC 3986. Message-ID: details: https://hg.nginx.org/nginx/rev/dfd8dfb436e5 branches: changeset: 7880:dfd8dfb436e5 user: Maxim Dounin date: Mon Jun 28 18:01:11 2021 +0300 description: Core: escaping of chars not allowed in URIs per RFC 3986. Per RFC 3986 only the following characters are allowed in URIs unescaped: unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "=" And "%" can appear as a part of escaping itself. The following characters are not allowed and need to be escaped: %00-%1F, %7F-%FF, " ", """, "<", ">", "\", "^", "`", "{", "|", "}". Not escaping ">" is known to cause problems at least with MS Exchange (see http://nginx.org/pipermail/nginx-ru/2010-January/031261.html) and in Tomcat (ticket #2191). The patch adds escaping of the following chars in all URI parts: """, "<", ">", "\", "^", "`", "{", "|", "}". Note that comments are mostly preserved to outline important characters being escaped. diffstat: src/core/ngx_string.c | 45 +++++++++++++++++++++++++++++---------------- 1 files changed, 29 insertions(+), 16 deletions(-) diffs (112 lines): diff -r b7407334c60d -r dfd8dfb436e5 src/core/ngx_string.c --- a/src/core/ngx_string.c Mon Jun 28 18:01:09 2021 +0300 +++ b/src/core/ngx_string.c Mon Jun 28 18:01:11 2021 +0300 @@ -1493,19 +1493,32 @@ ngx_escape_uri(u_char *dst, u_char *src, uint32_t *escape; static u_char hex[] = "0123456789ABCDEF"; - /* " ", "#", "%", "?", %00-%1F, %7F-%FF */ + /* + * Per RFC 3986 only the following chars are allowed in URIs unescaped: + * + * unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + * gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" + * sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + * / "*" / "+" / "," / ";" / "=" + * + * And "%" can appear as a part of escaping itself. The following + * characters are not allowed and need to be escaped: %00-%1F, %7F-%FF, + * " ", """, "<", ">", "\", "^", "`", "{", "|", "}". + */ + + /* " ", "#", "%", "?", not allowed */ static uint32_t uri[] = { 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ - 0x80000029, /* 1000 0000 0000 0000 0000 0000 0010 1001 */ + 0xd000002d, /* 1101 0000 0000 0000 0000 0000 0010 1101 */ /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ - 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x50000000, /* 0101 0000 0000 0000 0000 0000 0000 0000 */ /* ~}| {zyx wvut srqp onml kjih gfed cba` */ - 0x80000000, /* 1000 0000 0000 0000 0000 0000 0000 0000 */ + 0xb8000001, /* 1011 1000 0000 0000 0000 0000 0000 0001 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ @@ -1513,19 +1526,19 @@ ngx_escape_uri(u_char *dst, u_char *src, 0xffffffff /* 1111 1111 1111 1111 1111 1111 1111 1111 */ }; - /* " ", "#", "%", "&", "+", ";", "?", %00-%1F, %7F-%FF */ + /* " ", "#", "%", "&", "+", ";", "?", not allowed */ static uint32_t args[] = { 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ - 0x88000869, /* 1000 1000 0000 0000 0000 1000 0110 1001 */ + 0xd800086d, /* 1101 1000 0000 0000 0000 1000 0110 1101 */ /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ - 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x50000000, /* 0101 0000 0000 0000 0000 0000 0000 0000 */ /* ~}| {zyx wvut srqp onml kjih gfed cba` */ - 0x80000000, /* 1000 0000 0000 0000 0000 0000 0000 0000 */ + 0xb8000001, /* 1011 1000 0000 0000 0000 0000 0000 0001 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ @@ -1553,19 +1566,19 @@ ngx_escape_uri(u_char *dst, u_char *src, 0xffffffff /* 1111 1111 1111 1111 1111 1111 1111 1111 */ }; - /* " ", "#", """, "%", "'", %00-%1F, %7F-%FF */ + /* " ", "#", """, "%", "'", not allowed */ static uint32_t html[] = { 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ - 0x000000ad, /* 0000 0000 0000 0000 0000 0000 1010 1101 */ + 0x500000ad, /* 0101 0000 0000 0000 0000 0000 1010 1101 */ /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ - 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x50000000, /* 0101 0000 0000 0000 0000 0000 0000 0000 */ /* ~}| {zyx wvut srqp onml kjih gfed cba` */ - 0x80000000, /* 1000 0000 0000 0000 0000 0000 0000 0000 */ + 0xb8000001, /* 1011 1000 0000 0000 0000 0000 0000 0001 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ @@ -1573,19 +1586,19 @@ ngx_escape_uri(u_char *dst, u_char *src, 0xffffffff /* 1111 1111 1111 1111 1111 1111 1111 1111 */ }; - /* " ", """, "'", %00-%1F, %7F-%FF */ + /* " ", """, "'", not allowed */ static uint32_t refresh[] = { 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ - 0x00000085, /* 0000 0000 0000 0000 0000 0000 1000 0101 */ + 0x50000085, /* 0101 0000 0000 0000 0000 0000 1000 0101 */ /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ - 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x50000000, /* 0101 0000 0000 0000 0000 0000 0000 0000 */ /* ~}| {zyx wvut srqp onml kjih gfed cba` */ - 0x80000000, /* 1000 0000 0000 0000 0000 0000 0000 0000 */ + 0xd8000001, /* 1011 1000 0000 0000 0000 0000 0000 0001 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ From mdounin at mdounin.ru Mon Jun 28 18:36:30 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 28 Jun 2021 18:36:30 +0000 Subject: [nginx] Disabled spaces in URIs (ticket #196). Message-ID: details: https://hg.nginx.org/nginx/rev/52338ddf9e2f branches: changeset: 7881:52338ddf9e2f user: Maxim Dounin date: Mon Jun 28 18:01:13 2021 +0300 description: Disabled spaces in URIs (ticket #196). >From now on, requests with spaces in URIs are immediately rejected rather than allowed. Spaces were allowed in 31e9677b15a1 (0.8.41) to handle bad clients. It is believed that now this behaviour causes more harm than good. diffstat: src/http/modules/ngx_http_proxy_module.c | 4 +- src/http/ngx_http_parse.c | 72 +++---------------------------- src/http/ngx_http_request.c | 2 +- src/http/ngx_http_request.h | 3 - 4 files changed, 11 insertions(+), 70 deletions(-) diffs (199 lines): diff -r dfd8dfb436e5 -r 52338ddf9e2f src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c Mon Jun 28 18:01:11 2021 +0300 +++ b/src/http/modules/ngx_http_proxy_module.c Mon Jun 28 18:01:13 2021 +0300 @@ -1186,7 +1186,7 @@ ngx_http_proxy_create_key(ngx_http_reque loc_len = (r->valid_location && ctx->vars.uri.len) ? plcf->location.len : 0; - if (r->quoted_uri || r->space_in_uri || r->internal) { + if (r->quoted_uri || r->internal) { escape = 2 * ngx_escape_uri(NULL, r->uri.data + loc_len, r->uri.len - loc_len, NGX_ESCAPE_URI); } else { @@ -1299,7 +1299,7 @@ ngx_http_proxy_create_request(ngx_http_r loc_len = (r->valid_location && ctx->vars.uri.len) ? plcf->location.len : 0; - if (r->quoted_uri || r->space_in_uri || r->internal) { + if (r->quoted_uri || r->internal) { escape = 2 * ngx_escape_uri(NULL, r->uri.data + loc_len, r->uri.len - loc_len, NGX_ESCAPE_URI); } diff -r dfd8dfb436e5 -r 52338ddf9e2f src/http/ngx_http_parse.c --- a/src/http/ngx_http_parse.c Mon Jun 28 18:01:11 2021 +0300 +++ b/src/http/ngx_http_parse.c Mon Jun 28 18:01:13 2021 +0300 @@ -116,10 +116,8 @@ ngx_http_parse_request_line(ngx_http_req sw_host_end, sw_host_ip_literal, sw_port, - sw_host_http_09, sw_after_slash_in_uri, sw_check_uri, - sw_check_uri_http_09, sw_uri, sw_http_09, sw_http_H, @@ -398,7 +396,7 @@ ngx_http_parse_request_line(ngx_http_req */ r->uri_start = r->schema_end + 1; r->uri_end = r->schema_end + 2; - state = sw_host_http_09; + state = sw_http_09; break; default: return NGX_HTTP_PARSE_INVALID_REQUEST; @@ -472,35 +470,13 @@ ngx_http_parse_request_line(ngx_http_req */ r->uri_start = r->schema_end + 1; r->uri_end = r->schema_end + 2; - state = sw_host_http_09; + state = sw_http_09; break; default: return NGX_HTTP_PARSE_INVALID_REQUEST; } break; - /* space+ after "http://host[:port] " */ - case sw_host_http_09: - switch (ch) { - case ' ': - break; - case CR: - r->http_minor = 9; - state = sw_almost_done; - break; - case LF: - r->http_minor = 9; - goto done; - case 'H': - r->http_protocol.data = p; - state = sw_http_H; - break; - default: - return NGX_HTTP_PARSE_INVALID_REQUEST; - } - break; - - /* check "/.", "//", "%", and "\" (Win32) in URI */ case sw_after_slash_in_uri: @@ -512,7 +488,7 @@ ngx_http_parse_request_line(ngx_http_req switch (ch) { case ' ': r->uri_end = p; - state = sw_check_uri_http_09; + state = sw_http_09; break; case CR: r->uri_end = p; @@ -584,7 +560,7 @@ ngx_http_parse_request_line(ngx_http_req break; case ' ': r->uri_end = p; - state = sw_check_uri_http_09; + state = sw_http_09; break; case CR: r->uri_end = p; @@ -621,31 +597,6 @@ ngx_http_parse_request_line(ngx_http_req } break; - /* space+ after URI */ - case sw_check_uri_http_09: - switch (ch) { - case ' ': - break; - case CR: - r->http_minor = 9; - state = sw_almost_done; - break; - case LF: - r->http_minor = 9; - goto done; - case 'H': - r->http_protocol.data = p; - state = sw_http_H; - break; - default: - r->space_in_uri = 1; - state = sw_check_uri; - p--; - break; - } - break; - - /* URI */ case sw_uri: @@ -692,10 +643,7 @@ ngx_http_parse_request_line(ngx_http_req state = sw_http_H; break; default: - r->space_in_uri = 1; - state = sw_uri; - p--; - break; + return NGX_HTTP_PARSE_INVALID_REQUEST; } break; @@ -1171,9 +1119,7 @@ ngx_http_parse_uri(ngx_http_request_t *r switch (ch) { case ' ': - r->space_in_uri = 1; - state = sw_check_uri; - break; + return NGX_ERROR; case '.': r->complex_uri = 1; state = sw_uri; @@ -1232,8 +1178,7 @@ ngx_http_parse_uri(ngx_http_request_t *r r->uri_ext = p + 1; break; case ' ': - r->space_in_uri = 1; - break; + return NGX_ERROR; #if (NGX_WIN32) case '\\': r->complex_uri = 1; @@ -1267,8 +1212,7 @@ ngx_http_parse_uri(ngx_http_request_t *r switch (ch) { case ' ': - r->space_in_uri = 1; - break; + return NGX_ERROR; case '#': r->complex_uri = 1; break; diff -r dfd8dfb436e5 -r 52338ddf9e2f src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c Mon Jun 28 18:01:11 2021 +0300 +++ b/src/http/ngx_http_request.c Mon Jun 28 18:01:13 2021 +0300 @@ -1264,7 +1264,7 @@ ngx_http_process_request_uri(ngx_http_re r->unparsed_uri.len = r->uri_end - r->uri_start; r->unparsed_uri.data = r->uri_start; - r->valid_unparsed_uri = (r->space_in_uri || r->empty_path_in_uri) ? 0 : 1; + r->valid_unparsed_uri = r->empty_path_in_uri ? 0 : 1; if (r->uri_ext) { if (r->args_start) { diff -r dfd8dfb436e5 -r 52338ddf9e2f src/http/ngx_http_request.h --- a/src/http/ngx_http_request.h Mon Jun 28 18:01:11 2021 +0300 +++ b/src/http/ngx_http_request.h Mon Jun 28 18:01:13 2021 +0300 @@ -468,9 +468,6 @@ struct ngx_http_request_s { /* URI with "+" */ unsigned plus_in_uri:1; - /* URI with " " */ - unsigned space_in_uri:1; - /* URI with empty path */ unsigned empty_path_in_uri:1; From mdounin at mdounin.ru Mon Jun 28 18:36:32 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 28 Jun 2021 18:36:32 +0000 Subject: [nginx] Disabled control characters in URIs. Message-ID: details: https://hg.nginx.org/nginx/rev/b4073527be81 branches: changeset: 7882:b4073527be81 user: Maxim Dounin date: Mon Jun 28 18:01:15 2021 +0300 description: Disabled control characters in URIs. Control characters (0x00-0x1f, 0x7f) were never allowed in URIs, and must be percent-encoded by clients. Further, these are not believed to appear in practice. On the other hand, passing such characters might make various attacks possible or easier, despite the fact that currently allowed control characters are not significant for HTTP request parsing. diffstat: src/http/ngx_http_parse.c | 42 ++++++++++++++++++++++++++++-------------- 1 files changed, 28 insertions(+), 14 deletions(-) diffs (119 lines): diff -r 52338ddf9e2f -r b4073527be81 src/http/ngx_http_parse.c --- a/src/http/ngx_http_parse.c Mon Jun 28 18:01:13 2021 +0300 +++ b/src/http/ngx_http_parse.c Mon Jun 28 18:01:15 2021 +0300 @@ -11,7 +11,7 @@ static uint32_t usual[] = { - 0xffffdbfe, /* 1111 1111 1111 1111 1101 1011 1111 1110 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ 0x7fff37d6, /* 0111 1111 1111 1111 0011 0111 1101 0110 */ @@ -24,7 +24,7 @@ static uint32_t usual[] = { #endif /* ~}| {zyx wvut srqp onml kjih gfed cba` */ - 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0x7fffffff, /* 0111 1111 1111 1111 1111 1111 1111 1111 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ @@ -528,9 +528,10 @@ ngx_http_parse_request_line(ngx_http_req case '+': r->plus_in_uri = 1; break; - case '\0': - return NGX_HTTP_PARSE_INVALID_REQUEST; default: + if (ch < 0x20 || ch == 0x7f) { + return NGX_HTTP_PARSE_INVALID_REQUEST; + } state = sw_check_uri; break; } @@ -592,8 +593,11 @@ ngx_http_parse_request_line(ngx_http_req case '+': r->plus_in_uri = 1; break; - case '\0': - return NGX_HTTP_PARSE_INVALID_REQUEST; + default: + if (ch < 0x20 || ch == 0x7f) { + return NGX_HTTP_PARSE_INVALID_REQUEST; + } + break; } break; @@ -621,8 +625,11 @@ ngx_http_parse_request_line(ngx_http_req case '#': r->complex_uri = 1; break; - case '\0': - return NGX_HTTP_PARSE_INVALID_REQUEST; + default: + if (ch < 0x20 || ch == 0x7f) { + return NGX_HTTP_PARSE_INVALID_REQUEST; + } + break; } break; @@ -1118,8 +1125,6 @@ ngx_http_parse_uri(ngx_http_request_t *r } switch (ch) { - case ' ': - return NGX_ERROR; case '.': r->complex_uri = 1; state = sw_uri; @@ -1150,6 +1155,9 @@ ngx_http_parse_uri(ngx_http_request_t *r r->plus_in_uri = 1; break; default: + if (ch <= 0x20 || ch == 0x7f) { + return NGX_ERROR; + } state = sw_check_uri; break; } @@ -1177,8 +1185,6 @@ ngx_http_parse_uri(ngx_http_request_t *r case '.': r->uri_ext = p + 1; break; - case ' ': - return NGX_ERROR; #if (NGX_WIN32) case '\\': r->complex_uri = 1; @@ -1200,6 +1206,11 @@ ngx_http_parse_uri(ngx_http_request_t *r case '+': r->plus_in_uri = 1; break; + default: + if (ch <= 0x20 || ch == 0x7f) { + return NGX_ERROR; + } + break; } break; @@ -1211,11 +1222,14 @@ ngx_http_parse_uri(ngx_http_request_t *r } switch (ch) { - case ' ': - return NGX_ERROR; case '#': r->complex_uri = 1; break; + default: + if (ch <= 0x20 || ch == 0x7f) { + return NGX_ERROR; + } + break; } break; } From mdounin at mdounin.ru Mon Jun 28 18:36:35 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 28 Jun 2021 18:36:35 +0000 Subject: [nginx] Disabled control characters and space in header names. Message-ID: details: https://hg.nginx.org/nginx/rev/41f4bd4c51f1 branches: changeset: 7883:41f4bd4c51f1 user: Maxim Dounin date: Mon Jun 28 18:01:18 2021 +0300 description: Disabled control characters and space in header names. Control characters (0x00-0x1f, 0x7f), space, and colon were never allowed in header names. The only somewhat valid use is header continuation which nginx never supported and which is explicitly obsolete by RFC 7230. Previously, such headers were considered invalid and were ignored by default (as per ignore_invalid_headers directive). With this change, such headers are unconditionally rejected. It is expected to make nginx more resilient to various attacks, in particular, with ignore_invalid_headers switched off (which is inherently unsecure, though nevertheless sometimes used in the wild). diffstat: src/http/modules/ngx_http_grpc_module.c | 2 +- src/http/ngx_http_parse.c | 4 ++-- src/http/v2/ngx_http_v2.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diffs (45 lines): diff -r b4073527be81 -r 41f4bd4c51f1 src/http/modules/ngx_http_grpc_module.c --- a/src/http/modules/ngx_http_grpc_module.c Mon Jun 28 18:01:15 2021 +0300 +++ b/src/http/modules/ngx_http_grpc_module.c Mon Jun 28 18:01:18 2021 +0300 @@ -3384,7 +3384,7 @@ ngx_http_grpc_validate_header_name(ngx_h return NGX_ERROR; } - if (ch == '\0' || ch == CR || ch == LF) { + if (ch <= 0x20 || ch == 0x7f) { return NGX_ERROR; } } diff -r b4073527be81 -r 41f4bd4c51f1 src/http/ngx_http_parse.c --- a/src/http/ngx_http_parse.c Mon Jun 28 18:01:15 2021 +0300 +++ b/src/http/ngx_http_parse.c Mon Jun 28 18:01:18 2021 +0300 @@ -893,7 +893,7 @@ ngx_http_parse_header_line(ngx_http_requ break; } - if (ch == '\0') { + if (ch <= 0x20 || ch == 0x7f || ch == ':') { return NGX_HTTP_PARSE_INVALID_HEADER; } @@ -961,7 +961,7 @@ ngx_http_parse_header_line(ngx_http_requ break; } - if (ch == '\0') { + if (ch <= 0x20 || ch == 0x7f) { return NGX_HTTP_PARSE_INVALID_HEADER; } diff -r b4073527be81 -r 41f4bd4c51f1 src/http/v2/ngx_http_v2.c --- a/src/http/v2/ngx_http_v2.c Mon Jun 28 18:01:15 2021 +0300 +++ b/src/http/v2/ngx_http_v2.c Mon Jun 28 18:01:18 2021 +0300 @@ -3457,7 +3457,7 @@ ngx_http_v2_validate_header(ngx_http_req continue; } - if (ch == '\0' || ch == LF || ch == CR || ch == ':' + if (ch <= 0x20 || ch == 0x7f || ch == ':' || (ch >= 'A' && ch <= 'Z')) { ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, From mdounin at mdounin.ru Mon Jun 28 18:36:38 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 28 Jun 2021 18:36:38 +0000 Subject: [nginx] Improved logging of invalid headers. Message-ID: details: https://hg.nginx.org/nginx/rev/b87b7092cedb branches: changeset: 7884:b87b7092cedb user: Maxim Dounin date: Mon Jun 28 18:01:20 2021 +0300 description: Improved logging of invalid headers. In 71edd9192f24 logging of invalid headers which were rejected with the NGX_HTTP_PARSE_INVALID_HEADER error was restricted to just the "client sent invalid header line" message, without any attempts to log the header itself. This patch returns logging of the header up to the invalid character and the character itself. The r->header_end pointer is now properly set in all cases to make logging possible. The same logging is also introduced when parsing headers from upstream servers. diffstat: src/http/modules/ngx_http_fastcgi_module.c | 10 ++++++---- src/http/modules/ngx_http_proxy_module.c | 10 ++++++---- src/http/modules/ngx_http_scgi_module.c | 10 ++++++---- src/http/modules/ngx_http_uwsgi_module.c | 10 ++++++---- src/http/ngx_http_parse.c | 5 +++++ src/http/ngx_http_request.c | 4 +++- 6 files changed, 32 insertions(+), 17 deletions(-) diffs (137 lines): diff -r 41f4bd4c51f1 -r b87b7092cedb src/http/modules/ngx_http_fastcgi_module.c --- a/src/http/modules/ngx_http_fastcgi_module.c Mon Jun 28 18:01:18 2021 +0300 +++ b/src/http/modules/ngx_http_fastcgi_module.c Mon Jun 28 18:01:20 2021 +0300 @@ -2019,10 +2019,12 @@ ngx_http_fastcgi_process_header(ngx_http break; } - /* there was error while a header line parsing */ - - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "upstream sent invalid header"); + /* rc == NGX_HTTP_PARSE_INVALID_HEADER */ + + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "upstream sent invalid header: \"%*s\\x%02xd...\"", + r->header_end - r->header_name_start, + r->header_name_start, *r->header_end); return NGX_HTTP_UPSTREAM_INVALID_HEADER; } diff -r 41f4bd4c51f1 -r b87b7092cedb src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c Mon Jun 28 18:01:18 2021 +0300 +++ b/src/http/modules/ngx_http_proxy_module.c Mon Jun 28 18:01:20 2021 +0300 @@ -2019,10 +2019,12 @@ ngx_http_proxy_process_header(ngx_http_r return NGX_AGAIN; } - /* there was error while a header line parsing */ - - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "upstream sent invalid header"); + /* rc == NGX_HTTP_PARSE_INVALID_HEADER */ + + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "upstream sent invalid header: \"%*s\\x%02xd...\"", + r->header_end - r->header_name_start, + r->header_name_start, *r->header_end); return NGX_HTTP_UPSTREAM_INVALID_HEADER; } diff -r 41f4bd4c51f1 -r b87b7092cedb src/http/modules/ngx_http_scgi_module.c --- a/src/http/modules/ngx_http_scgi_module.c Mon Jun 28 18:01:18 2021 +0300 +++ b/src/http/modules/ngx_http_scgi_module.c Mon Jun 28 18:01:20 2021 +0300 @@ -1140,10 +1140,12 @@ ngx_http_scgi_process_header(ngx_http_re return NGX_AGAIN; } - /* there was error while a header line parsing */ - - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "upstream sent invalid header"); + /* rc == NGX_HTTP_PARSE_INVALID_HEADER */ + + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "upstream sent invalid header: \"%*s\\x%02xd...\"", + r->header_end - r->header_name_start, + r->header_name_start, *r->header_end); return NGX_HTTP_UPSTREAM_INVALID_HEADER; } diff -r 41f4bd4c51f1 -r b87b7092cedb src/http/modules/ngx_http_uwsgi_module.c --- a/src/http/modules/ngx_http_uwsgi_module.c Mon Jun 28 18:01:18 2021 +0300 +++ b/src/http/modules/ngx_http_uwsgi_module.c Mon Jun 28 18:01:20 2021 +0300 @@ -1361,10 +1361,12 @@ ngx_http_uwsgi_process_header(ngx_http_r return NGX_AGAIN; } - /* there was error while a header line parsing */ - - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "upstream sent invalid header"); + /* rc == NGX_HTTP_PARSE_INVALID_HEADER */ + + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "upstream sent invalid header: \"%*s\\x%02xd...\"", + r->header_end - r->header_name_start, + r->header_name_start, *r->header_end); return NGX_HTTP_UPSTREAM_INVALID_HEADER; } diff -r 41f4bd4c51f1 -r b87b7092cedb src/http/ngx_http_parse.c --- a/src/http/ngx_http_parse.c Mon Jun 28 18:01:18 2021 +0300 +++ b/src/http/ngx_http_parse.c Mon Jun 28 18:01:20 2021 +0300 @@ -894,6 +894,7 @@ ngx_http_parse_header_line(ngx_http_requ } if (ch <= 0x20 || ch == 0x7f || ch == ':') { + r->header_end = p; return NGX_HTTP_PARSE_INVALID_HEADER; } @@ -962,6 +963,7 @@ ngx_http_parse_header_line(ngx_http_requ } if (ch <= 0x20 || ch == 0x7f) { + r->header_end = p; return NGX_HTTP_PARSE_INVALID_HEADER; } @@ -984,6 +986,7 @@ ngx_http_parse_header_line(ngx_http_requ r->header_end = p; goto done; case '\0': + r->header_end = p; return NGX_HTTP_PARSE_INVALID_HEADER; default: r->header_start = p; @@ -1007,6 +1010,7 @@ ngx_http_parse_header_line(ngx_http_requ r->header_end = p; goto done; case '\0': + r->header_end = p; return NGX_HTTP_PARSE_INVALID_HEADER; } break; @@ -1022,6 +1026,7 @@ ngx_http_parse_header_line(ngx_http_requ case LF: goto done; case '\0': + r->header_end = p; return NGX_HTTP_PARSE_INVALID_HEADER; default: state = sw_value; diff -r 41f4bd4c51f1 -r b87b7092cedb src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c Mon Jun 28 18:01:18 2021 +0300 +++ b/src/http/ngx_http_request.c Mon Jun 28 18:01:20 2021 +0300 @@ -1522,7 +1522,9 @@ ngx_http_process_request_headers(ngx_eve /* rc == NGX_HTTP_PARSE_INVALID_HEADER */ ngx_log_error(NGX_LOG_INFO, c->log, 0, - "client sent invalid header line"); + "client sent invalid header line: \"%*s\\x%02xd...\"", + r->header_end - r->header_name_start, + r->header_name_start, *r->header_end); ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); break; From mdounin at mdounin.ru Mon Jun 28 18:36:41 2021 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 28 Jun 2021 18:36:41 +0000 Subject: [nginx] Disabled control characters in the Host header. Message-ID: details: https://hg.nginx.org/nginx/rev/e0fdd75871e4 branches: changeset: 7885:e0fdd75871e4 user: Maxim Dounin date: Mon Jun 28 18:01:24 2021 +0300 description: Disabled control characters in the Host header. Control characters (0x00-0x1f, 0x7f) and space are not expected to appear in the Host header. Requests with such characters in the Host header are now unconditionally rejected. diffstat: src/http/ngx_http_request.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diffs (23 lines): diff -r b87b7092cedb -r e0fdd75871e4 src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c Mon Jun 28 18:01:20 2021 +0300 +++ b/src/http/ngx_http_request.c Mon Jun 28 18:01:24 2021 +0300 @@ -2176,15 +2176,16 @@ ngx_http_validate_host(ngx_str_t *host, } break; - case '\0': - return NGX_DECLINED; - default: if (ngx_path_separator(ch)) { return NGX_DECLINED; } + if (ch <= 0x20 || ch == 0x7f) { + return NGX_DECLINED; + } + if (ch >= 'A' && ch <= 'Z') { alloc = 1; } From xeioex at nginx.com Tue Jun 29 12:13:37 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 29 Jun 2021 12:13:37 +0000 Subject: [njs] Fixed parsing of export default declarations. Message-ID: details: https://hg.nginx.org/njs/rev/f1a1b9ad198d branches: changeset: 1668:f1a1b9ad198d user: Artem S. Povalyukhin date: Thu Jun 24 08:53:12 2021 +0300 description: Fixed parsing of export default declarations. With non-assignment expression. diffstat: src/njs_parser.c | 6 +++++- test/module/export_non_assignment.js | 1 + test/njs_expect_test.exp | 2 ++ 3 files changed, 8 insertions(+), 1 deletions(-) diffs (40 lines): diff -r f10d5c38f098 -r f1a1b9ad198d src/njs_parser.c --- a/src/njs_parser.c Fri Jun 25 17:00:12 2021 +0000 +++ b/src/njs_parser.c Thu Jun 24 08:53:12 2021 +0300 @@ -7468,7 +7468,7 @@ njs_parser_export(njs_parser_t *parser, node->token_line = parser->line; parser->node = node; - njs_parser_next(parser, njs_parser_expression); + njs_parser_next(parser, njs_parser_assignment_expression); return njs_parser_after(parser, current, node, 1, njs_parser_export_after); } @@ -7478,6 +7478,10 @@ static njs_int_t njs_parser_export_after(njs_parser_t *parser, njs_lexer_token_t *token, njs_queue_link_t *current) { + if (njs_parser_expect_semicolon(parser, token) != NJS_OK) { + return njs_parser_failed(parser); + } + parser->target->right = parser->node; parser->node = parser->target; diff -r f10d5c38f098 -r f1a1b9ad198d test/module/export_non_assignment.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/module/export_non_assignment.js Thu Jun 24 08:53:12 2021 +0300 @@ -0,0 +1,1 @@ +export default 10, 11; diff -r f10d5c38f098 -r f1a1b9ad198d test/njs_expect_test.exp --- a/test/njs_expect_test.exp Fri Jun 25 17:00:12 2021 +0000 +++ b/test/njs_expect_test.exp Thu Jun 24 08:53:12 2021 +0300 @@ -749,6 +749,8 @@ njs_test { "Identifier \"default\" has already been declared in export.js:5\r\n"} {"import m from 'export_non_default.js'\r\n" "Non-default export is not supported in export_non_default.js:3\r\n"} + {"import m from 'export_non_assignment.js'\r\n" + "Unexpected token \",\" in export_non_assignment.js:1\r\n"} {"import ref from 'ref_exception.js'\r\n" "ReferenceError: \"undeclared\" is not defined"} {"var ref\r\n" From xeioex at nginx.com Tue Jun 29 12:13:39 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 29 Jun 2021 12:13:39 +0000 Subject: [njs] Version 0.6.1. Message-ID: details: https://hg.nginx.org/njs/rev/4adbe67b292a branches: changeset: 1669:4adbe67b292a user: Dmitry Volyntsev date: Tue Jun 29 11:47:04 2021 +0000 description: Version 0.6.1. diffstat: CHANGES | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diffs (16 lines): diff -r f1a1b9ad198d -r 4adbe67b292a CHANGES --- a/CHANGES Thu Jun 24 08:53:12 2021 +0300 +++ b/CHANGES Tue Jun 29 11:47:04 2021 +0000 @@ -1,3 +1,12 @@ +Changes with njs 0.6.1 29 Jun 2021 + + *) Bugfix: fixed RegExpBuiltinExec() with UTF-8 only regexps. + The bug was introduced in 0.4.2. + + *) Bugfix: fixed parsing of export default declaration with + non-assignment expressions. + Thanks to Artem S. Povalyukhin. + Changes with njs 0.6.0 15 Jun 2021 Core: From xeioex at nginx.com Tue Jun 29 12:13:41 2021 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 29 Jun 2021 12:13:41 +0000 Subject: [njs] Added tag 0.6.1 for changeset 4adbe67b292a Message-ID: details: https://hg.nginx.org/njs/rev/0c5715790b37 branches: changeset: 1670:0c5715790b37 user: Dmitry Volyntsev date: Tue Jun 29 12:12:57 2021 +0000 description: Added tag 0.6.1 for changeset 4adbe67b292a diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 4adbe67b292a -r 0c5715790b37 .hgtags --- a/.hgtags Tue Jun 29 11:47:04 2021 +0000 +++ b/.hgtags Tue Jun 29 12:12:57 2021 +0000 @@ -43,3 +43,4 @@ d355071f55ef4612d89db0ba72e7aaeaa99deef7 e5de01378b1a8ab0a94dd3a8c4c6bb7a235f4b9c 0.5.2 282b9412976ceee31eb12876f1499fe975e6f08c 0.5.3 742ebceef2b5d15febc093172fe6174e427b26c8 0.6.0 +4adbe67b292af2adc0a6fde4ec6cb95dbba9470a 0.6.1 From alexander.borisov at nginx.com Tue Jun 29 14:09:05 2021 From: alexander.borisov at nginx.com (Alexander Borisov) Date: Tue, 29 Jun 2021 14:09:05 +0000 Subject: [njs] Version bump. Message-ID: details: https://hg.nginx.org/njs/rev/fe1888cc2e3c branches: changeset: 1671:fe1888cc2e3c user: Alexander Borisov date: Tue Jun 29 17:08:21 2021 +0300 description: Version bump. diffstat: src/njs.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 0c5715790b37 -r fe1888cc2e3c src/njs.h --- a/src/njs.h Tue Jun 29 12:12:57 2021 +0000 +++ b/src/njs.h Tue Jun 29 17:08:21 2021 +0300 @@ -11,7 +11,7 @@ #include -#define NJS_VERSION "0.6.1" +#define NJS_VERSION "0.6.2" #include /* STDOUT_FILENO, STDERR_FILENO */ From alexander.borisov at nginx.com Tue Jun 29 14:09:07 2021 From: alexander.borisov at nginx.com (Alexander Borisov) Date: Tue, 29 Jun 2021 14:09:07 +0000 Subject: [njs] Fixed rest parameter parsing without binding identifier. Message-ID: details: https://hg.nginx.org/njs/rev/45c470a2d710 branches: changeset: 1672:45c470a2d710 user: Alexander Borisov date: Tue Jun 29 17:08:23 2021 +0300 description: Fixed rest parameter parsing without binding identifier. This closes #400 issue on GitHub. diffstat: src/njs_parser.c | 9 +++++++++ src/test/njs_unit_test.c | 6 ++++++ 2 files changed, 15 insertions(+), 0 deletions(-) diffs (35 lines): diff -r fe1888cc2e3c -r 45c470a2d710 src/njs_parser.c --- a/src/njs_parser.c Tue Jun 29 17:08:21 2021 +0300 +++ b/src/njs_parser.c Tue Jun 29 17:08:23 2021 +0300 @@ -6791,6 +6791,15 @@ njs_parser_formal_parameters(njs_parser_ njs_lexer_consume_token(parser->lexer, 1); + token = njs_lexer_token(parser->lexer, 0); + if (token == NULL) { + return NJS_ERROR; + } + + if (!njs_lexer_token_is_binding_identifier(token)) { + return njs_parser_failed(parser); + } + lambda->rest_parameters = 1; return NJS_OK; diff -r fe1888cc2e3c -r 45c470a2d710 src/test/njs_unit_test.c --- a/src/test/njs_unit_test.c Tue Jun 29 17:08:21 2021 +0300 +++ b/src/test/njs_unit_test.c Tue Jun 29 17:08:23 2021 +0300 @@ -9330,6 +9330,12 @@ static njs_unit_test_t njs_test[] = { njs_str("function f(a,...rest) { }; f.length"), njs_str("1") }, + { njs_str("function f(...) {}"), + njs_str("SyntaxError: Unexpected token \")\" in 1") }, + + { njs_str("(function (...) {})()"), + njs_str("SyntaxError: Unexpected token \")\" in 1") }, + { njs_str("function f(a,b) { }; var ff = f.bind(f, 1); ff.length"), njs_str("1") },