From david.freedman at uk.clara.net Mon Oct 2 15:29:55 2017 From: david.freedman at uk.clara.net (David Freedman) Date: Mon, 2 Oct 2017 15:29:55 +0000 Subject: Contrib: http2 per server (was re: [nginx] support http2 per server) In-Reply-To: <091DFB29-F91A-4C72-B629-20342A542E90@uk.clara.net> References: <091DFB29-F91A-4C72-B629-20342A542E90@uk.clara.net> Message-ID: Not that anybody has responded yet, but please find an important improvement over this patch: - if (hc->addr_conf->http2 && !sscf->h2) { + if (r->http_version == NGX_HTTP_VERSION_20 && !sscf->h2) { Full patch (with this improvement included) below: diff -r 6b6e15bbda92 -r 2806e0ba8e91 src/http/modules/ngx_http_ssl_module.c --- a/src/http/modules/ngx_http_ssl_module.c Tue Sep 05 17:59:31 2017 +0300 +++ b/src/http/modules/ngx_http_ssl_module.c Fri Sep 08 01:07:46 2017 +0000 @@ -234,6 +234,13 @@ offsetof(ngx_http_ssl_srv_conf_t, stapling_verify), NULL }, + { ngx_string("ssl_h2"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, + ngx_http_ssl_enable, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_ssl_srv_conf_t, h2), + NULL }, + ngx_null_command }; @@ -354,6 +361,7 @@ #endif #if (NGX_HTTP_V2) ngx_http_connection_t *hc; + ngx_http_ssl_srv_conf_t *sscf; #endif #if (NGX_HTTP_V2 || NGX_DEBUG) ngx_connection_t *c; @@ -372,7 +380,9 @@ #if (NGX_HTTP_V2) hc = c->data; - if (hc->addr_conf->http2) { + sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_ssl_module); + + if (hc->addr_conf->http2 && sscf->h2) { srv = (unsigned char *) NGX_HTTP_V2_ALPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; srvlen = sizeof(NGX_HTTP_V2_ALPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1; @@ -416,10 +426,13 @@ #if (NGX_HTTP_V2) { ngx_http_connection_t *hc; + ngx_http_ssl_srv_conf_t *sscf; hc = c->data; - if (hc->addr_conf->http2) { + sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_ssl_module); + + if (hc->addr_conf->http2 && sscf->h2) { *out = (unsigned char *) NGX_HTTP_V2_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; *outlen = sizeof(NGX_HTTP_V2_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1; @@ -559,6 +572,7 @@ sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; sscf->stapling = NGX_CONF_UNSET; sscf->stapling_verify = NGX_CONF_UNSET; + sscf->h2 = NGX_CONF_UNSET; return sscf; } @@ -624,6 +638,8 @@ ngx_conf_merge_str_value(conf->stapling_responder, prev->stapling_responder, ""); + ngx_conf_merge_value(conf->h2, prev->h2, 1); + conf->ssl.log = cf->log; if (conf->enable) { diff -r 6b6e15bbda92 -r 2806e0ba8e91 src/http/modules/ngx_http_ssl_module.h --- a/src/http/modules/ngx_http_ssl_module.h Tue Sep 05 17:59:31 2017 +0300 +++ b/src/http/modules/ngx_http_ssl_module.h Fri Sep 08 01:07:46 2017 +0000 @@ -57,6 +57,9 @@ u_char *file; ngx_uint_t line; + + ngx_flag_t h2; + } ngx_http_ssl_srv_conf_t; diff -r 6b6e15bbda92 -r 2806e0ba8e91 src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c Tue Sep 05 17:59:31 2017 +0300 +++ b/src/http/ngx_http_request.c Fri Sep 08 01:07:46 2017 +0000 @@ -795,6 +795,7 @@ unsigned int len; const unsigned char *data; ngx_http_connection_t *hc; + ngx_http_ssl_srv_conf_t *sscf; hc = c->data; @@ -813,9 +814,15 @@ SSL_get0_next_proto_negotiated(c->ssl->connection, &data, &len); #endif - if (len == 2 && data[0] == 'h' && data[1] == '2') { - ngx_http_v2_init(c->read); - return; + sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_ssl_module); + + if (sscf->h2) { + + if (len == 2 && data[0] == 'h' && data[1] == '2') { + ngx_http_v2_init(c->read); + return; + } + } } } @@ -2106,6 +2113,15 @@ ngx_http_finalize_request(r, NGX_HTTP_MISDIRECTED_REQUEST); return NGX_ERROR; } +#if (NGX_HTTP_V2) + if (r->http_version == NGX_HTTP_VERSION_20 && !sscf->h2) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client attempted to request a server name " + "that does not have http2 enabled"); + ngx_http_finalize_request(r, NGX_HTTP_MISDIRECTED_REQUEST); + return NGX_ERROR; + } +#endif } #endif From mdounin at mdounin.ru Mon Oct 2 16:08:30 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 02 Oct 2017 16:08:30 +0000 Subject: [nginx] MIME: style. Message-ID: details: http://hg.nginx.org/nginx/rev/b31caae8cc7b branches: changeset: 7114:b31caae8cc7b user: Maxim Dounin date: Mon Oct 02 19:06:51 2017 +0300 description: MIME: style. Restored alphabetical order within groups, OOXML types placed to the application/ group and wrapped to avoid lines longer than 80 chars. diffstat: conf/mime.types | 18 ++++++++++-------- 1 files changed, 10 insertions(+), 8 deletions(-) diffs (51 lines): diff --git a/conf/mime.types b/conf/mime.types --- a/conf/mime.types +++ b/conf/mime.types @@ -16,13 +16,13 @@ types { text/x-component htc; image/png png; + image/svg+xml svg svgz; image/tiff tif tiff; image/vnd.wap.wbmp wbmp; + image/webp webp; image/x-icon ico; image/x-jng jng; image/x-ms-bmp bmp; - image/svg+xml svg svgz; - image/webp webp; application/font-woff woff; application/java-archive jar war ear; @@ -33,12 +33,18 @@ types { application/postscript ps eps ai; application/rtf rtf; application/vnd.apple.mpegurl m3u8; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; application/vnd.ms-excel xls; application/vnd.ms-fontobject eot; application/vnd.ms-powerpoint ppt; + application/vnd.openxmlformats-officedocument.presentationml.presentation + pptx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet + xlsx; + application/vnd.openxmlformats-officedocument.wordprocessingml.document + docx; application/vnd.wap.wmlc wmlc; - application/vnd.google-earth.kml+xml kml; - application/vnd.google-earth.kmz kmz; application/x-7z-compressed 7z; application/x-cocoa cco; application/x-java-archive-diff jardiff; @@ -64,10 +70,6 @@ types { application/octet-stream iso img; application/octet-stream msi msp msm; - application/vnd.openxmlformats-officedocument.wordprocessingml.document docx; - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx; - application/vnd.openxmlformats-officedocument.presentationml.presentation pptx; - audio/midi mid midi kar; audio/mpeg mp3; audio/ogg ogg; From mdounin at mdounin.ru Mon Oct 2 16:08:32 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 02 Oct 2017 16:08:32 +0000 Subject: [nginx] MIME: added most common OpenDocument types. Message-ID: details: http://hg.nginx.org/nginx/rev/4ff31c785d0c branches: changeset: 7115:4ff31c785d0c user: Maxim Dounin date: Mon Oct 02 19:07:01 2017 +0300 description: MIME: added most common OpenDocument types. Requested by Michiel Leenaars. diffstat: conf/mime.types | 162 ++++++++++++++++++++++++++++--------------------------- 1 files changed, 83 insertions(+), 79 deletions(-) diffs (178 lines): diff --git a/conf/mime.types b/conf/mime.types --- a/conf/mime.types +++ b/conf/mime.types @@ -1,91 +1,95 @@ types { - text/html html htm shtml; - text/css css; - text/xml xml; - image/gif gif; - image/jpeg jpeg jpg; - application/javascript js; - application/atom+xml atom; - application/rss+xml rss; + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + application/atom+xml atom; + application/rss+xml rss; - text/mathml mml; - text/plain txt; - text/vnd.sun.j2me.app-descriptor jad; - text/vnd.wap.wml wml; - text/x-component htc; + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; - image/png png; - image/svg+xml svg svgz; - image/tiff tif tiff; - image/vnd.wap.wbmp wbmp; - image/webp webp; - image/x-icon ico; - image/x-jng jng; - image/x-ms-bmp bmp; + image/png png; + image/svg+xml svg svgz; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/webp webp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; - application/font-woff woff; - application/java-archive jar war ear; - application/json json; - application/mac-binhex40 hqx; - application/msword doc; - application/pdf pdf; - application/postscript ps eps ai; - application/rtf rtf; - application/vnd.apple.mpegurl m3u8; - application/vnd.google-earth.kml+xml kml; - application/vnd.google-earth.kmz kmz; - application/vnd.ms-excel xls; - application/vnd.ms-fontobject eot; - application/vnd.ms-powerpoint ppt; + application/font-woff woff; + application/java-archive jar war ear; + application/json json; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.apple.mpegurl m3u8; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/vnd.ms-excel xls; + application/vnd.ms-fontobject eot; + application/vnd.ms-powerpoint ppt; + application/vnd.oasis.opendocument.graphics odg; + application/vnd.oasis.opendocument.presentation odp; + application/vnd.oasis.opendocument.spreadsheet ods; + application/vnd.oasis.opendocument.text odt; application/vnd.openxmlformats-officedocument.presentationml.presentation - pptx; + pptx; application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - xlsx; + xlsx; application/vnd.openxmlformats-officedocument.wordprocessingml.document - docx; - application/vnd.wap.wmlc wmlc; - application/x-7z-compressed 7z; - application/x-cocoa cco; - application/x-java-archive-diff jardiff; - application/x-java-jnlp-file jnlp; - application/x-makeself run; - application/x-perl pl pm; - application/x-pilot prc pdb; - application/x-rar-compressed rar; - application/x-redhat-package-manager rpm; - application/x-sea sea; - application/x-shockwave-flash swf; - application/x-stuffit sit; - application/x-tcl tcl tk; - application/x-x509-ca-cert der pem crt; - application/x-xpinstall xpi; - application/xhtml+xml xhtml; - application/xspf+xml xspf; - application/zip zip; + docx; + application/vnd.wap.wmlc wmlc; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xspf+xml xspf; + application/zip zip; - application/octet-stream bin exe dll; - application/octet-stream deb; - application/octet-stream dmg; - application/octet-stream iso img; - application/octet-stream msi msp msm; + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream iso img; + application/octet-stream msi msp msm; - audio/midi mid midi kar; - audio/mpeg mp3; - audio/ogg ogg; - audio/x-m4a m4a; - audio/x-realaudio ra; + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-m4a m4a; + audio/x-realaudio ra; - video/3gpp 3gpp 3gp; - video/mp2t ts; - video/mp4 mp4; - video/mpeg mpeg mpg; - video/quicktime mov; - video/webm webm; - video/x-flv flv; - video/x-m4v m4v; - video/x-mng mng; - video/x-ms-asf asx asf; - video/x-ms-wmv wmv; - video/x-msvideo avi; + video/3gpp 3gpp 3gp; + video/mp2t ts; + video/mp4 mp4; + video/mpeg mpeg mpg; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-m4v m4v; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; } From mdounin at mdounin.ru Mon Oct 2 16:56:33 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 02 Oct 2017 16:56:33 +0000 Subject: [nginx] Upstream: better handling of invalid headers in cache files. Message-ID: details: http://hg.nginx.org/nginx/rev/46ddff109e72 branches: changeset: 7116:46ddff109e72 user: Maxim Dounin date: Mon Oct 02 19:10:20 2017 +0300 description: Upstream: better handling of invalid headers in cache files. If cache file is truncated, it is possible that u->process_header() will return NGX_AGAIN. Added appropriate handling of this case by changing the error to NGX_HTTP_UPSTREAM_INVALID_HEADER. Also, added appropriate logging of this and NGX_HTTP_UPSTREAM_INVALID_HEADER cases at the "crit" level. Note that this will result in duplicate logging in case of NGX_HTTP_UPSTREAM_INVALID_HEADER. While this is something better to avoid, it is considered to be an overkill to implement cache-specific error logging in u->process_header(). Additionally, u->buffer.start is now reset to be able to receive a new response, and u->cache_status set to MISS to provide the value in the $upstream_cache_status variable, much like it happens on other cache file errors detected by ngx_http_file_cache_read(), instead of HIT, which is believed to be misleading. diffstat: src/http/ngx_http_upstream.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diffs (29 lines): diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -582,6 +582,8 @@ ngx_http_upstream_init_request(ngx_http_ if (rc == NGX_HTTP_UPSTREAM_INVALID_HEADER) { rc = NGX_DECLINED; r->cached = 0; + u->buffer.start = NULL; + u->cache_status = NGX_HTTP_CACHE_MISS; } if (ngx_http_upstream_cache_background_update(r, u) != NGX_OK) { @@ -1059,8 +1061,16 @@ ngx_http_upstream_cache_send(ngx_http_re return NGX_ERROR; } + if (rc == NGX_AGAIN) { + rc = NGX_HTTP_UPSTREAM_INVALID_HEADER; + } + /* rc == NGX_HTTP_UPSTREAM_INVALID_HEADER */ + ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0, + "cache file \"%s\" contains invalid header", + c->file.name.data); + /* TODO: delete file */ return rc; From mdounin at mdounin.ru Tue Oct 3 16:21:05 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 03 Oct 2017 16:21:05 +0000 Subject: [nginx] Cache: fixed caching of intercepted errors (ticket #1382). Message-ID: details: http://hg.nginx.org/nginx/rev/dbd77a638eb7 branches: changeset: 7117:dbd77a638eb7 user: Maxim Dounin date: Tue Oct 03 18:19:27 2017 +0300 description: Cache: fixed caching of intercepted errors (ticket #1382). When caching intercepted errors, previous behaviour was to use proxy_cache_valid times specified, regardless of various cache control headers present in the response. Fix is to check u->cacheable and use u->cache->valid_sec as set by various cache control response headers, similar to how we do this in the normal caching code path. diffstat: src/http/ngx_http_upstream.c | 24 +++++++++++++++++------- 1 files changed, 17 insertions(+), 7 deletions(-) diffs (34 lines): diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -2528,13 +2528,23 @@ ngx_http_upstream_intercept_errors(ngx_h #if (NGX_HTTP_CACHE) if (r->cache) { - time_t valid; - - valid = ngx_http_file_cache_valid(u->conf->cache_valid, status); - - if (valid) { - r->cache->valid_sec = ngx_time() + valid; - r->cache->error = status; + + if (u->cacheable) { + time_t valid; + + valid = r->cache->valid_sec; + + if (valid == 0) { + valid = ngx_http_file_cache_valid(u->conf->cache_valid, + status); + if (valid) { + r->cache->valid_sec = ngx_time() + valid; + } + } + + if (valid) { + r->cache->error = status; + } } ngx_http_file_cache_free(r->cache, u->pipe->temp_file); From xeioex at nginx.com Tue Oct 3 18:25:29 2017 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Tue, 03 Oct 2017 18:25:29 +0000 Subject: [njs] JSON object. Message-ID: details: http://hg.nginx.org/njs/rev/94c42736a730 branches: changeset: 410:94c42736a730 user: Dmitry Volyntsev date: Tue Oct 03 21:24:58 2017 +0300 description: JSON object. diffstat: Makefile | 14 + njs/njs_array.c | 2 +- njs/njs_array.h | 1 + njs/njs_builtin.c | 2 + njs/njs_generator.c | 1 + njs/njs_json.c | 2088 ++++++++++++++++++++++++++++++++++++++++++++++ njs/njs_json.h | 14 + njs/njs_lexer_keyword.c | 1 + njs/njs_number.c | 51 +- njs/njs_number.h | 1 + njs/njs_object.c | 42 +- njs/njs_object.h | 2 +- njs/njs_object_hash.h | 10 + njs/njs_parser.c | 1 + njs/njs_parser.h | 1 + njs/njs_vm.h | 3 +- njs/test/njs_unit_test.c | 589 ++++++++++++ nxt/nxt_utf8.h | 30 + 18 files changed, 2815 insertions(+), 38 deletions(-) diffs (truncated from 3094 to 1000 lines): diff -r f6b9efd315c5 -r 94c42736a730 Makefile --- a/Makefile Tue Sep 26 14:19:49 2017 +0300 +++ b/Makefile Tue Oct 03 21:24:58 2017 +0300 @@ -16,6 +16,7 @@ NXT_BUILDDIR = build $(NXT_BUILDDIR)/njs_string.o \ $(NXT_BUILDDIR)/njs_object.o \ $(NXT_BUILDDIR)/njs_array.o \ + $(NXT_BUILDDIR)/njs_json.o \ $(NXT_BUILDDIR)/njs_function.o \ $(NXT_BUILDDIR)/njs_regexp.o \ $(NXT_BUILDDIR)/njs_date.o \ @@ -48,6 +49,7 @@ NXT_BUILDDIR = build $(NXT_BUILDDIR)/njs_string.o \ $(NXT_BUILDDIR)/njs_object.o \ $(NXT_BUILDDIR)/njs_array.o \ + $(NXT_BUILDDIR)/njs_json.o \ $(NXT_BUILDDIR)/njs_function.o \ $(NXT_BUILDDIR)/njs_regexp.o \ $(NXT_BUILDDIR)/njs_date.o \ @@ -211,6 +213,18 @@ dist: -I$(NXT_LIB) -Injs \ njs/njs_array.c +$(NXT_BUILDDIR)/njs_json.o: \ + $(NXT_BUILDDIR)/libnxt.a \ + njs/njscript.h \ + njs/njs_vm.h \ + njs/njs_object.h \ + njs/njs_json.c \ + + $(NXT_CC) -c -o $(NXT_BUILDDIR)/njs_json.o $(NXT_CFLAGS) \ + -I$(NXT_LIB) -Injs \ + njs/njs_json.c + + $(NXT_BUILDDIR)/njs_function.o: \ $(NXT_BUILDDIR)/libnxt.a \ njs/njscript.h \ diff -r f6b9efd315c5 -r 94c42736a730 njs/njs_array.c --- a/njs/njs_array.c Tue Sep 26 14:19:49 2017 +0300 +++ b/njs/njs_array.c Tue Oct 03 21:24:58 2017 +0300 @@ -157,7 +157,7 @@ njs_array_alloc(njs_vm_t *vm, uint32_t l } -static njs_ret_t +njs_ret_t njs_array_add(njs_vm_t *vm, njs_array_t *array, njs_value_t *value) { njs_ret_t ret; diff -r f6b9efd315c5 -r 94c42736a730 njs/njs_array.h --- a/njs/njs_array.h Tue Sep 26 14:19:49 2017 +0300 +++ b/njs/njs_array.h Tue Oct 03 21:24:58 2017 +0300 @@ -16,6 +16,7 @@ njs_array_t *njs_array_alloc(njs_vm_t *vm, uint32_t length, uint32_t spare); +njs_ret_t njs_array_add(njs_vm_t *vm, njs_array_t *array, njs_value_t *value); njs_ret_t njs_array_string_add(njs_vm_t *vm, njs_array_t *array, u_char *start, size_t size, size_t length); njs_ret_t njs_array_expand(njs_vm_t *vm, njs_array_t *array, uint32_t prepend, diff -r f6b9efd315c5 -r 94c42736a730 njs/njs_builtin.c --- a/njs/njs_builtin.c Tue Sep 26 14:19:49 2017 +0300 +++ b/njs/njs_builtin.c Tue Oct 03 21:24:58 2017 +0300 @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,7 @@ static nxt_int_t njs_builtin_completions const njs_object_init_t *njs_object_init[] = { NULL, /* global this */ &njs_math_object_init, /* Math */ + &njs_json_object_init, /* JSON */ }; diff -r f6b9efd315c5 -r 94c42736a730 njs/njs_generator.c --- a/njs/njs_generator.c Tue Sep 26 14:19:49 2017 +0300 +++ b/njs/njs_generator.c Tue Oct 03 21:24:58 2017 +0300 @@ -298,6 +298,7 @@ njs_generator(njs_vm_t *vm, njs_parser_t case NJS_TOKEN_GLOBAL_THIS: case NJS_TOKEN_MATH: + case NJS_TOKEN_JSON: case NJS_TOKEN_EVAL: case NJS_TOKEN_TO_STRING: case NJS_TOKEN_IS_NAN: diff -r f6b9efd315c5 -r 94c42736a730 njs/njs_json.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/njs/njs_json.c Tue Oct 03 21:24:58 2017 +0300 @@ -0,0 +1,2088 @@ + +/* + * Copyright (C) Dmitry Volyntsev + * Copyright (C) NGINX, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +typedef struct { + njs_vm_t *vm; + nxt_mem_cache_pool_t *pool; + nxt_uint_t depth; + u_char *start; + u_char *end; +} njs_json_parse_ctx_t; + + +typedef struct { + njs_value_t value; + + uint8_t written; /* 1 bit */ + + enum { + NJS_JSON_OBJECT_START, + NJS_JSON_OBJECT_CONTINUE, + NJS_JSON_OBJECT_TO_JSON_REPLACED, + NJS_JSON_OBJECT_REPLACED, + NJS_JSON_ARRAY_START, + NJS_JSON_ARRAY_CONTINUE, + NJS_JSON_ARRAY_TO_JSON_REPLACED, + NJS_JSON_ARRAY_REPLACED + } type:8; + + uint32_t index; + njs_array_t *keys; + njs_value_t *prop_value; +} njs_json_state_t; + + +typedef struct { + union { + njs_continuation_t cont; + u_char padding[NJS_CONTINUATION_SIZE]; + } u; + /* + * This retval value must be aligned so the continuation is padded + * to aligned size. + */ + njs_value_t retval; + + nxt_array_t stack; + njs_json_state_t *state; + njs_function_t *function; +} njs_json_parse_t; + + +typedef struct njs_chb_node_s njs_chb_node_t; + +struct njs_chb_node_s { + njs_chb_node_t *next; + u_char *start; + u_char *pos; + u_char *end; +}; + + +typedef struct { + union { + njs_continuation_t cont; + u_char padding[NJS_CONTINUATION_SIZE]; + } u; + /* + * This retval value must be aligned so the continuation is padded + * to aligned size. + */ + njs_value_t retval; + njs_value_t key; + + njs_vm_t *vm; + nxt_mem_cache_pool_t *pool; + njs_chb_node_t *nodes; + njs_chb_node_t *last; + nxt_array_t stack; + njs_json_state_t *state; + + njs_value_t replacer; + nxt_str_t space; +} njs_json_stringify_t; + + +static u_char *njs_json_parse_value(njs_json_parse_ctx_t *ctx, + njs_value_t *value, u_char *p); +static u_char *njs_json_parse_object(njs_json_parse_ctx_t *ctx, + njs_value_t *value, u_char *p); +static u_char *njs_json_parse_array(njs_json_parse_ctx_t *ctx, + njs_value_t *value, u_char *p); +static u_char *njs_json_parse_string(njs_json_parse_ctx_t *ctx, + njs_value_t *value, u_char *p); +static u_char *njs_json_parse_number(njs_json_parse_ctx_t *ctx, + njs_value_t *value, u_char *p); +nxt_inline uint32_t njs_json_unicode(const u_char *p); +static u_char *njs_json_skip_space(u_char *start, u_char *end); + +static njs_ret_t njs_json_parse_continuation(njs_vm_t *vm, + njs_value_t *args, nxt_uint_t nargs, njs_index_t unused); +static njs_ret_t njs_json_parse_continuation_apply(njs_vm_t *vm, + njs_json_parse_t *parse); +static njs_json_state_t *njs_json_push_parse_state(njs_vm_t *vm, + njs_json_parse_t *parse, njs_value_t *value); +static njs_json_state_t *njs_json_pop_parse_state(njs_json_parse_t *parse); +static void njs_json_parse_exception(njs_json_parse_ctx_t *ctx, + const char* msg, u_char *pos); + +static njs_ret_t njs_json_stringify_continuation(njs_vm_t *vm, + njs_value_t *args, nxt_uint_t nargs, njs_index_t unused); +static njs_function_t *njs_object_to_json_function(njs_vm_t *vm, + njs_value_t *value); +static njs_ret_t njs_json_stringify_to_json(njs_vm_t *vm, + njs_json_stringify_t* stringify, njs_function_t *function, + njs_value_t *key, njs_value_t *value); +static njs_ret_t njs_json_stringify_replacer(njs_vm_t *vm, + njs_json_stringify_t* stringify, njs_value_t *key, njs_value_t *value); +static njs_ret_t njs_json_stringify_array(njs_vm_t *vm, + njs_json_stringify_t *stringify); +static njs_json_state_t *njs_json_push_stringify_state(njs_vm_t *vm, + njs_json_stringify_t *stringify, njs_value_t *value); +static njs_json_state_t *njs_json_pop_stringify_state( + njs_json_stringify_t *stringify); +static void njs_json_stringify_exception(njs_json_stringify_t *stringify, + const char* msg); + +static nxt_int_t njs_json_append_value(njs_json_stringify_t *stringify, + njs_value_t *value); +static nxt_int_t njs_json_append_string(njs_json_stringify_t *stringify, + njs_value_t *value); +static nxt_int_t njs_json_append_number(njs_json_stringify_t *stringify, + njs_value_t *value); + +static njs_value_t *njs_json_wrap_value(njs_vm_t *vm, njs_value_t *value); + + +#define NJS_JSON_BUF_MIN_SIZE 128 + +#define njs_json_buf_written(stringify, bytes) \ + (stringify)->last->pos += (bytes); + +#define njs_json_buf_node_size(n) (size_t) ((n)->pos - (n)->start) +#define njs_json_buf_node_room(n) (size_t) ((n)->end - (n)->pos) + +static nxt_int_t njs_json_buf_append(njs_json_stringify_t *stringify, + const char* msg, size_t len); +static u_char *njs_json_buf_reserve(njs_json_stringify_t *stringify, + size_t size); +static nxt_int_t njs_json_buf_pullup(njs_json_stringify_t *stringify, + nxt_str_t *str); + + +static njs_ret_t +njs_json_parse(njs_vm_t *vm, njs_value_t *args, nxt_uint_t nargs, + njs_index_t unused) +{ + u_char *p, *end; + njs_value_t arg, *value, *wrapper; + njs_json_parse_t *parse; + njs_string_prop_t string; + njs_json_parse_ctx_t ctx; + + value = nxt_mem_cache_alloc(vm->mem_cache_pool, sizeof(njs_value_t)); + if (nxt_slow_path(value == NULL)) { + vm->exception = &njs_exception_memory_error; + return NXT_ERROR; + } + + if (nargs < 2) { + arg = njs_string_void; + } else { + arg = args[1]; + } + + (void) njs_string_prop(&string, &arg); + + p = string.start; + end = p + string.size; + + ctx.vm = vm; + ctx.pool = vm->mem_cache_pool; + ctx.depth = 32; + ctx.start = string.start; + ctx.end = end; + + p = njs_json_skip_space(p, end); + if (nxt_slow_path(p == end)) { + njs_json_parse_exception(&ctx, "Unexpected end of input", p); + return NXT_ERROR; + } + + p = njs_json_parse_value(&ctx, value, p); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + p = njs_json_skip_space(p, end); + if (nxt_slow_path(p != end)) { + njs_json_parse_exception(&ctx, "Unexpected token", p); + return NXT_ERROR; + } + + if (nargs >= 3 && njs_is_function(&args[2]) && njs_is_object(value)) { + wrapper = njs_json_wrap_value(vm, value); + if (nxt_slow_path(wrapper == NULL)) { + goto memory_error; + } + + parse = njs_vm_continuation(vm); + parse->u.cont.function = njs_json_parse_continuation; + parse->function = args[2].data.u.function; + + if (nxt_array_init(&parse->stack, NULL, 4, sizeof(njs_json_state_t), + &njs_array_mem_proto, vm->mem_cache_pool) + == NULL) + { + goto memory_error; + } + + if (njs_json_push_parse_state(vm, parse, wrapper) == NULL) { + goto memory_error; + } + + return njs_json_parse_continuation(vm, args, nargs, unused); + } + + vm->retval = *value; + + return NXT_OK; + +memory_error: + + vm->exception = &njs_exception_memory_error; + return NXT_ERROR; +} + + +static njs_ret_t +njs_json_stringify(njs_vm_t *vm, njs_value_t *args, nxt_uint_t nargs, + njs_index_t unused) +{ + double num; + nxt_int_t i; + njs_ret_t ret; + njs_value_t *wrapper; + njs_json_stringify_t *stringify; + + if (nargs < 2) { + vm->retval = njs_value_void; + return NXT_OK; + } + + stringify = njs_vm_continuation(vm); + stringify->vm = vm; + stringify->pool = vm->mem_cache_pool; + stringify->u.cont.function = njs_json_stringify_continuation; + stringify->nodes = NULL; + stringify->last = NULL; + + if (nargs >= 3 && (njs_is_function(&args[2]) || njs_is_array(&args[2]))) { + stringify->replacer = args[2]; + if (njs_is_array(&args[2])) { + ret = njs_json_stringify_array(vm, stringify); + if (nxt_slow_path(ret != NXT_OK)) { + goto memory_error; + } + } + + } else { + stringify->replacer = njs_value_void; + } + + stringify->space.length = 0; + + if (nargs >= 4 && (njs_is_string(&args[3]) || njs_is_number(&args[3]))) { + if (njs_is_string(&args[3])) { + njs_string_get(&args[3], &stringify->space); + stringify->space.length = nxt_min(stringify->space.length, 10); + + } else { + num = args[3].data.u.number; + if (!isnan(num) && !isinf(num) && num > 0) { + num = nxt_min(num, 10); + + stringify->space.length = (size_t) num; + stringify->space.start = nxt_mem_cache_alloc(vm->mem_cache_pool, + (size_t) num + 1); + if (nxt_slow_path(stringify->space.start == NULL)) { + goto memory_error; + } + + for (i = 0; i < (int) num; i++) { + stringify->space.start[i] = ' '; + } + } + } + } + + if (nxt_array_init(&stringify->stack, NULL, 4, sizeof(njs_json_state_t), + &njs_array_mem_proto, vm->mem_cache_pool) + == NULL) + { + goto memory_error; + } + + wrapper = njs_json_wrap_value(vm, &args[1]); + if (nxt_slow_path(wrapper == NULL)) { + goto memory_error; + } + + if (njs_json_push_stringify_state(vm, stringify, wrapper) == NULL) { + goto memory_error; + } + + return njs_json_stringify_continuation(vm, args, nargs, unused); + +memory_error: + + vm->exception = &njs_exception_memory_error; + return NXT_ERROR; +} + + +static u_char * +njs_json_parse_value(njs_json_parse_ctx_t *ctx, njs_value_t *value, u_char *p) +{ + switch (*p) { + case '{': + return njs_json_parse_object(ctx, value, p); + + case '[': + return njs_json_parse_array(ctx, value, p); + + case '"': + return njs_json_parse_string(ctx, value, p); + + case 't': + if (nxt_fast_path(ctx->end - p >= 4 && memcmp(p, "true", 4) == 0)) { + *value = njs_value_true; + + return p + 4; + } + + goto error; + + case 'f': + if (nxt_fast_path(ctx->end - p >= 5 && memcmp(p, "false", 5) == 0)) { + *value = njs_value_false; + + return p + 5; + } + + goto error; + + case 'n': + if (nxt_fast_path(ctx->end - p >= 4 && memcmp(p, "null", 4) == 0)) { + *value = njs_value_null; + + return p + 4; + } + + goto error; + } + + if (nxt_fast_path(*p == '-' || (*p - '0') <= 9)) { + return njs_json_parse_number(ctx, value, p); + } + +error: + + njs_json_parse_exception(ctx, "Unexpected token", p); + + return NULL; +} + + +static u_char * +njs_json_parse_object(njs_json_parse_ctx_t *ctx, njs_value_t *value, u_char *p) +{ + nxt_int_t ret; + njs_object_t *object; + njs_value_t *prop_name, *prop_value; + njs_object_prop_t *prop; + nxt_lvlhsh_query_t lhq; + + if (nxt_slow_path(--ctx->depth == 0)) { + njs_json_parse_exception(ctx, "Nested too deep", p); + return NULL; + } + + object = njs_object_alloc(ctx->vm); + if (nxt_slow_path(object == NULL)) { + goto memory_error; + } + + prop = NULL; + + for ( ;; ) { + p = njs_json_skip_space(p + 1, ctx->end); + if (nxt_slow_path(p == ctx->end)) { + goto error_end; + } + + if (*p != '"') { + if (nxt_fast_path(*p == '}')) { + if (nxt_slow_path(prop != NULL)) { + njs_json_parse_exception(ctx, "Trailing comma", p - 1); + return NULL; + } + + break; + } + + goto error_token; + } + + prop_name = nxt_mem_cache_alloc(ctx->pool, sizeof(njs_value_t)); + if (nxt_slow_path(prop_name == NULL)) { + goto memory_error; + } + + p = njs_json_parse_string(ctx, prop_name, p); + if (nxt_slow_path(p == NULL)) { + /* The exception is set by the called function. */ + return NULL; + } + + p = njs_json_skip_space(p, ctx->end); + if (nxt_slow_path(p == ctx->end || *p != ':')) { + goto error_token; + } + + p = njs_json_skip_space(p + 1, ctx->end); + if (nxt_slow_path(p == ctx->end)) { + goto error_end; + } + + prop_value = nxt_mem_cache_alloc(ctx->pool, sizeof(njs_value_t)); + if (nxt_slow_path(prop_value == NULL)) { + goto memory_error; + } + + p = njs_json_parse_value(ctx, prop_value, p); + if (nxt_slow_path(p == NULL)) { + /* The exception is set by the called function. */ + return NULL; + } + + prop = njs_object_prop_alloc(ctx->vm, prop_name, prop_value, 1); + if (nxt_slow_path(prop == NULL)) { + goto memory_error; + } + + njs_string_get(prop_name, &lhq.key); + lhq.key_hash = nxt_djb_hash(lhq.key.start, lhq.key.length); + lhq.value = prop; + lhq.replace = 1; + lhq.pool = ctx->pool; + lhq.proto = &njs_object_hash_proto; + + ret = nxt_lvlhsh_insert(&object->hash, &lhq); + if (nxt_slow_path(ret != NXT_OK)) { + ctx->vm->exception = &njs_exception_internal_error; + return NULL; + } + + p = njs_json_skip_space(p, ctx->end); + if (nxt_slow_path(p == ctx->end)) { + goto error_end; + } + + if (*p != ',') { + if (nxt_fast_path(*p == '}')) { + break; + } + + goto error_token; + } + } + + value->data.u.object = object; + value->type = NJS_OBJECT; + value->data.truth = 1; + + ctx->depth++; + + return p + 1; + +error_token: + + njs_json_parse_exception(ctx, "Unexpected token", p); + + return NULL; + +error_end: + + njs_json_parse_exception(ctx, "Unexpected end of input", p); + + return NULL; + +memory_error: + + ctx->vm->exception = &njs_exception_memory_error; + + return NULL; +} + + +static u_char * +njs_json_parse_array(njs_json_parse_ctx_t *ctx, njs_value_t *value, u_char *p) +{ + nxt_int_t ret; + njs_array_t *array; + njs_value_t *element; + + if (nxt_slow_path(--ctx->depth == 0)) { + njs_json_parse_exception(ctx, "Nested too deep", p); + return NULL; + } + + array = njs_array_alloc(ctx->vm, 0, 0); + if (nxt_slow_path(array == NULL)) { + ctx->vm->exception = &njs_exception_memory_error; + return NULL; + } + + element = NULL; + + for ( ;; ) { + p = njs_json_skip_space(p + 1, ctx->end); + if (nxt_slow_path(p == ctx->end)) { + goto error_end; + } + + if (*p == ']') { + if (nxt_slow_path(element != NULL)) { + njs_json_parse_exception(ctx, "Trailing comma", p - 1); + return NULL; + } + + break; + } + + element = nxt_mem_cache_alloc(ctx->pool, sizeof(njs_value_t)); + if (nxt_slow_path(element == NULL)) { + ctx->vm->exception = &njs_exception_memory_error; + return NULL; + } + + p = njs_json_parse_value(ctx, element, p); + if (nxt_slow_path(p == NULL)) { + return NULL; + } + + ret = njs_array_add(ctx->vm, array, element); + if (nxt_slow_path(ret != NXT_OK)) { + ctx->vm->exception = &njs_exception_internal_error; + return NULL; + } + + p = njs_json_skip_space(p, ctx->end); + if (nxt_slow_path(p == ctx->end)) { + goto error_end; + } + + if (*p != ',') { + if (nxt_fast_path(*p == ']')) { + break; + } + + goto error_token; + } + } + + value->data.u.array = array; + value->type = NJS_ARRAY; + value->data.truth = 1; + + ctx->depth++; + + return p + 1; + +error_token: + + njs_json_parse_exception(ctx, "Unexpected token", p); + + return NULL; + +error_end: + + njs_json_parse_exception(ctx, "Unexpected end of input", p); + + return NULL; +} + + +static u_char * +njs_json_parse_string(njs_json_parse_ctx_t *ctx, njs_value_t *value, u_char *p) +{ + u_char *s, ch, *last, *start; + size_t size, surplus; + ssize_t length; + uint32_t utf, utf_low; + njs_ret_t ret; + + enum { + sw_usual = 0, + sw_escape, + sw_encoded1, + sw_encoded2, + sw_encoded3, + sw_encoded4, + } state; + + start = p + 1; + + state = 0; + surplus = 0; + + for (p = start; p < ctx->end; p++) { + ch = *p; + + switch (state) { + + case sw_usual: + + if (ch == '"') { + break; + } + + if (ch == '\\') { + state = sw_escape; + continue; + } + + if (nxt_fast_path(ch >= ' ')) { + continue; + } + + njs_json_parse_exception(ctx, "Forbidden source char", p); + + return NULL; + + case sw_escape: + + switch (ch) { + case '"': + case '\\': + case '/': + case 'n': + case 'r': + case 't': + case 'b': + case 'f': + surplus++; + state = sw_usual; + continue; + + case 'u': + /* + * Basic unicode 6 bytes "\uXXXX" in JSON + * and up to 3 bytes in UTF-8. + * + * Surrogate pair: 12 bytes "\uXXXX\uXXXX" in JSON + * and 3 or 4 bytes in UTF-8. + */ + surplus += 3; + state = sw_encoded1; + continue; + } + + njs_json_parse_exception(ctx, "Unknown escape char", p); + + return NULL; + + case sw_encoded1: + case sw_encoded2: + case sw_encoded3: + case sw_encoded4: + + if (nxt_fast_path((ch >= '0' && ch <= '9') + || (ch >= 'A' && ch <= 'F') + || (ch >= 'a' && ch <= 'f'))) + { + state = (state == sw_encoded4) ? sw_usual : state + 1; + continue; + } + + njs_json_parse_exception(ctx, "Invalid Unicode escape sequence", p); + + return NULL; + } + + break; + } + + if (nxt_slow_path(p == ctx->end)) { + njs_json_parse_exception(ctx, "Unexpected end of input", p); + return NULL; + } + + /* Points to the ending quote mark. */ + last = p; + + size = last - start - surplus; + + if (surplus != 0) { + p = start; + + start = nxt_mem_cache_alloc(ctx->pool, size); + if (nxt_slow_path(start == NULL)) { + ctx->vm->exception = &njs_exception_memory_error; + return NULL; + } + + s = start; + + do { + ch = *p++; + + if (ch != '\\') { + *s++ = ch; + continue; + } + + ch = *p++; + + switch (ch) { + case '"': + case '\\': + case '/': + *s++ = ch; + continue; + + case 'n': + *s++ = '\n'; + continue; + + case 'r': + *s++ = '\r'; + continue; + + case 't': + *s++ = '\t'; + continue; + + case 'b': + *s++ = '\b'; + continue; + + case 'f': + *s++ = '\f'; + continue; + } + + /* "\uXXXX": Unicode escape sequence. */ + + utf = njs_json_unicode(p); + p += 4; + + if (utf >= 0xd800 && utf <= 0xdfff) { + + /* Surrogate pair. */ + + if (utf > 0xdbff || p[0] != '\\' || p[1] != 'u') { + njs_json_parse_exception(ctx, "Invalid Unicode char", p); + return NULL; + } + + p += 2; + + utf_low = njs_json_unicode(p); + p += 4; + + if (nxt_slow_path(utf_low < 0xdc00 || utf_low > 0xdfff)) { + njs_json_parse_exception(ctx, "Invalid surrogate pair", p); + return NULL; + } + + utf = 0x10000 + ((utf - 0xd800) << 10) + (utf_low - 0xdc00); + } + + s = nxt_utf8_encode(s, utf); + + } while (p != last); + + size = s - start; + } + + length = nxt_utf8_length(start, size); + if (nxt_slow_path(length < 0)) { + length = 0; + } + + ret = njs_string_create(ctx->vm, value, start, size, length); + if (nxt_slow_path(ret != NXT_OK)) { + ctx->vm->exception = &njs_exception_memory_error; + return NULL; + } + + return last + 1; +} + + +static u_char * +njs_json_parse_number(njs_json_parse_ctx_t *ctx, njs_value_t *value, u_char *p) +{ + u_char *start; + double num; + nxt_int_t sign; + + sign = 1; + + if (*p == '-') { + if (p + 1 == ctx->end) { + goto error; + } + + p++; + sign = -1; + } + + start = p; + num = njs_number_dec_parse(&p, ctx->end); + if (p != start) { + *value = njs_value_zero; + value->data.u.number = sign * num; + + return p; + } + +error: + + njs_json_parse_exception(ctx, "Unexpected number", p); + + return NULL; +} + + +nxt_inline uint32_t +njs_json_unicode(const u_char *p) +{ + u_char c; + uint32_t utf; + nxt_uint_t i; + + utf = 0; + + for (i = 0; i < 4; i++) { + utf <<= 4; + c = p[i] | 0x20; + c -= '0'; + if (c > 9) { + c += '0' - 'a' + 10; + } + + utf |= c; + } + + return utf; +} + + +static u_char * +njs_json_skip_space(u_char *start, u_char *end) +{ + u_char *p; + + for (p = start; nxt_fast_path(p != end); p++) { + + switch (*p) { + case ' ': + case '\t': + case '\r': + case '\n': + continue; + } + + break; From xeioex at nginx.com Wed Oct 4 15:58:40 2017 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Wed, 04 Oct 2017 15:58:40 +0000 Subject: [njs] Interactive shell: object level completions. Message-ID: details: http://hg.nginx.org/njs/rev/d45f6f0079ab branches: changeset: 411:d45f6f0079ab user: Dmitry Volyntsev date: Wed Oct 04 18:58:10 2017 +0300 description: Interactive shell: object level completions. diffstat: Makefile | 7 +- njs/njs.c | 89 +++++++++------ njs/njs_builtin.c | 247 ++++++++++++++++++++++++++++++++++++++---- njs/njscript.h | 2 +- njs/test/njs_expect_test.exp | 168 +++++++++++++++++++++++++++++ nxt/auto/configure | 1 + nxt/auto/expect | 31 +++++ 7 files changed, 477 insertions(+), 68 deletions(-) diffs (762 lines): diff -r 94c42736a730 -r d45f6f0079ab Makefile --- a/Makefile Tue Oct 03 21:24:58 2017 +0300 +++ b/Makefile Wed Oct 04 18:58:10 2017 +0300 @@ -78,13 +78,14 @@ all: test lib_test njs: $(NXT_BUILDDIR)/njs -test: \ +njs_interactive_test: njs_expect_test $(NXT_BUILDDIR)/njs_interactive_test + $(NXT_BUILDDIR)/njs_interactive_test + +test: njs_interactive_test \ $(NXT_BUILDDIR)/njs_unit_test \ - $(NXT_BUILDDIR)/njs_interactive_test \ $(NXT_BUILDDIR)/njs_benchmark \ $(NXT_BUILDDIR)/njs_unit_test d - $(NXT_BUILDDIR)/njs_interactive_test clean: rm -rf $(NXT_BUILDDIR) diff -r 94c42736a730 -r d45f6f0079ab njs/njs.c --- a/njs/njs.c Tue Oct 03 21:24:58 2017 +0300 +++ b/njs/njs.c Wed Oct 04 18:58:10 2017 +0300 @@ -50,7 +50,8 @@ typedef struct { size_t index; size_t length; njs_vm_t *vm; - const char **completions; + nxt_array_t *completions; + nxt_array_t *suffix_completions; nxt_lvlhsh_each_t lhe; njs_completion_phase_t phase; } njs_completion_t; @@ -257,8 +258,7 @@ njs_interactive_shell(njs_opts_t *opts, printf("interactive njscript\n\n"); - printf("v -> the properties of v object.\n"); - printf("v. -> all the available prototype methods.\n"); + printf("v. -> the properties and prototype methods of v.\n"); printf("type console.help() for more information\n\n"); for ( ;; ) { @@ -487,7 +487,7 @@ njs_editline_init(njs_vm_t *vm) rl_attempted_completion_function = njs_completion_handler; rl_basic_word_break_characters = (char *) " \t\n\"\\'`@$><=;,|&{("; - njs_completion.completions = njs_vm_completions(vm); + njs_completion.completions = njs_vm_completions(vm, NULL); if (njs_completion.completions == NULL) { return NXT_ERROR; } @@ -507,12 +507,18 @@ njs_completion_handler(const char *text, } +/* editline frees the buffer every time. */ +#define njs_editline(s) strndup((char *) (s)->start, (s)->length) + +#define njs_completion(c, i) &(((nxt_str_t *) (c)->start)[i]) + static char * njs_completion_generator(const char *text, int state) { char *completion; size_t len; - const char *name, *p; + nxt_str_t expression, *suffix; + const char *p; njs_variable_t *var; njs_completion_t *cmpl; @@ -528,20 +534,20 @@ njs_completion_generator(const char *tex if (cmpl->phase == NJS_COMPLETION_GLOBAL) { for ( ;; ) { - name = cmpl->completions[cmpl->index]; - if (name == NULL) { + if (cmpl->index >= cmpl->completions->items) { break; } - cmpl->index++; + suffix = njs_completion(cmpl->completions, cmpl->index++); - if (name[0] == '.') { + if (suffix->start[0] == '.' || suffix->length < cmpl->length) { continue; } - if (strncmp(name, text, cmpl->length) == 0) { - /* editline frees the buffer every time. */ - return strdup(name); + if (strncmp(text, (char *) suffix->start, + nxt_min(suffix->length, cmpl->length)) == 0) + { + return njs_editline(suffix); } } @@ -549,22 +555,14 @@ njs_completion_generator(const char *tex for ( ;; ) { var = nxt_lvlhsh_each(&cmpl->vm->parser->scope->variables, &cmpl->lhe); - if (var == NULL) { + if (var == NULL || var->name.length < cmpl->length) { break; } - if (strncmp((char *) var->name.start, text, cmpl->length) - == 0) + if (strncmp(text, (char *) var->name.start, + nxt_min(var->name.length, cmpl->length)) == 0) { - completion = malloc(var->name.length + 1); - if (completion == NULL) { - return NULL; - } - - memcpy(completion, var->name.start, var->name.length); - completion[var->name.length] = '\0'; - - return completion; + return njs_editline(&var->name); } } } @@ -573,11 +571,30 @@ njs_completion_generator(const char *tex return NULL; } + /* Getting the longest prefix before a '.' */ + + p = &text[cmpl->length - 1]; + while (p > text && *p != '.') { p--; } + + if (*p != '.') { + return NULL; + } + + expression.start = (u_char *) text; + expression.length = p - text; + + cmpl->suffix_completions = njs_vm_completions(cmpl->vm, &expression); + if (cmpl->suffix_completions == NULL) { + return NULL; + } + cmpl->index = 0; cmpl->phase = NJS_COMPLETION_SUFFIX; } - len = 1; + /* Getting the right-most suffix after a '.' */ + + len = 0; p = &text[cmpl->length - 1]; while (p > text && *p != '.') { @@ -585,31 +602,29 @@ njs_completion_generator(const char *tex len++; } - if (*p != '.') { - return NULL; - } + p++; for ( ;; ) { - name = cmpl->completions[cmpl->index++]; - if (name == NULL) { + if (cmpl->index >= cmpl->suffix_completions->items) { break; } - if (name[0] != '.') { + suffix = njs_completion(cmpl->suffix_completions, cmpl->index++); + + if (len != 0 && strncmp((char *) suffix->start, p, + nxt_min(len, suffix->length)) != 0) + { continue; } - if (strncmp(name, p, len) != 0) { - continue; - } - - len = strlen(name) + (p - text) + 2; + len = suffix->length + (p - text) + 1; completion = malloc(len); if (completion == NULL) { return NULL; } - snprintf(completion, len, "%.*s%s", (int) (p - text), text, name); + snprintf(completion, len, "%.*s%.*s", (int) (p - text), text, + (int) suffix->length, suffix->start); return completion; } diff -r 94c42736a730 -r d45f6f0079ab njs/njs_builtin.c --- a/njs/njs_builtin.c Tue Oct 03 21:24:58 2017 +0300 +++ b/njs/njs_builtin.c Wed Oct 04 18:58:10 2017 +0300 @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -39,7 +40,10 @@ typedef struct { static nxt_int_t njs_builtin_completions(njs_vm_t *vm, size_t *size, - const char **completions); + nxt_str_t *completions); +static nxt_array_t *njs_vm_expression_completions(njs_vm_t *vm, + nxt_str_t *expression); +static nxt_array_t *njs_object_completions(njs_vm_t *vm, njs_object_t *object); const njs_object_init_t *njs_object_init[] = { @@ -346,33 +350,40 @@ njs_builtin_objects_clone(njs_vm_t *vm) } -const char ** -njs_vm_completions(njs_vm_t *vm) +nxt_array_t * +njs_vm_completions(njs_vm_t *vm, nxt_str_t *expression) { - size_t size; - const char **completions; + size_t size; + nxt_array_t *completions; + + if (expression == NULL) { + if (njs_builtin_completions(vm, &size, NULL) != NXT_OK) { + return NULL; + } + + completions = nxt_array_create(size, sizeof(nxt_str_t), + &njs_array_mem_proto, + vm->mem_cache_pool); - if (njs_builtin_completions(vm, &size, NULL) != NXT_OK) { - return NULL; + if (nxt_slow_path(completions == NULL)) { + return NULL; + } + + if (njs_builtin_completions(vm, &size, completions->start) != NXT_OK) { + return NULL; + } + + completions->items = size; + + return completions; } - completions = nxt_mem_cache_zalloc(vm->mem_cache_pool, - sizeof(char *) * (size + 1)); - - if (completions == NULL) { - return NULL; - } - - if (njs_builtin_completions(vm, NULL, completions) != NXT_OK) { - return NULL; - } - - return completions; + return njs_vm_expression_completions(vm, expression); } static nxt_int_t -njs_builtin_completions(njs_vm_t *vm, size_t *size, const char **completions) +njs_builtin_completions(njs_vm_t *vm, size_t *size, nxt_str_t *completions) { char *compl; size_t n, len; @@ -398,7 +409,7 @@ njs_builtin_completions(njs_vm_t *vm, si } if (completions != NULL) { - completions[n++] = (char *) keyword->name.start; + completions[n++] = keyword->name; } else { n++; @@ -433,7 +444,8 @@ njs_builtin_completions(njs_vm_t *vm, si snprintf(compl, len, "%s.%s", njs_object_init[i]->name.start, string.start); - completions[n++] = (char *) compl; + completions[n].length = len; + completions[n++].start = (u_char *) compl; } else { n++; @@ -465,13 +477,16 @@ njs_builtin_completions(njs_vm_t *vm, si snprintf(compl, len, ".%s", string.start); for (k = 0; k < n; k++) { - if (strncmp(completions[k], compl, len) == 0) { + if (strncmp((char *) completions[k].start, compl, len) + == 0) + { break; } } if (k == n) { - completions[n++] = (char *) compl; + completions[n].length = len; + completions[n++].start = (u_char *) compl; } } else { @@ -504,7 +519,8 @@ njs_builtin_completions(njs_vm_t *vm, si snprintf(compl, len, "%s.%s", njs_constructor_init[i]->name.start, string.start); - completions[n++] = (char *) compl; + completions[n].length = len; + completions[n++].start = (u_char *) compl; } else { n++; @@ -533,7 +549,8 @@ njs_builtin_completions(njs_vm_t *vm, si snprintf(compl, len, "%.*s", (int) ext_object->name.length, ext_object->name.start); - completions[n++] = (char *) compl; + completions[n].length = len; + completions[n++].start = (u_char *) compl; } else { n++; @@ -557,7 +574,8 @@ njs_builtin_completions(njs_vm_t *vm, si (int) ext_object->name.length, ext_object->name.start, (int) ext_prop->name.length, ext_prop->name.start); - completions[n++] = (char *) compl; + completions[n].length = len; + completions[n++].start = (u_char *) compl; } else { n++; @@ -573,6 +591,181 @@ njs_builtin_completions(njs_vm_t *vm, si } +static nxt_array_t * +njs_vm_expression_completions(njs_vm_t *vm, nxt_str_t *expression) +{ + u_char *p, *end; + nxt_int_t ret; + njs_value_t *value; + njs_variable_t *var; + njs_object_prop_t *prop; + nxt_lvlhsh_query_t lhq; + + if (nxt_slow_path(vm->parser == NULL)) { + return NULL; + } + + p = expression->start; + end = p + expression->length; + + lhq.key.start = p; + + while (p < end && *p != '.') { p++; } + + lhq.proto = &njs_variables_hash_proto; + lhq.key.length = p - lhq.key.start; + lhq.key_hash = nxt_djb_hash(lhq.key.start, lhq.key.length); + + ret = nxt_lvlhsh_find(&vm->parser->scope->variables, &lhq); + if (nxt_slow_path(ret != NXT_OK)) { + return NULL; + } + + var = lhq.value; + value = njs_vmcode_operand(vm, var->index); + + if (!njs_is_object(value)) { + return NULL; + } + + lhq.proto = &njs_object_hash_proto; + + for ( ;; ) { + + if (p == end) { + break; + } + + lhq.key.start = ++p; + + while (p < end && *p != '.') { p++; } + + lhq.key.length = p - lhq.key.start; + lhq.key_hash = nxt_djb_hash(lhq.key.start, lhq.key.length); + + ret = nxt_lvlhsh_find(&value->data.u.object->hash, &lhq); + if (nxt_slow_path(ret != NXT_OK)) { + return NULL; + } + + prop = lhq.value; + + if (!njs_is_object(&prop->value)) { + return NULL; + } + + value = &prop->value; + } + + return njs_object_completions(vm, value->data.u.object); +} + + +static nxt_array_t * +njs_object_completions(njs_vm_t *vm, njs_object_t *object) +{ + size_t size; + nxt_uint_t n, k; + nxt_str_t *compl; + nxt_array_t *completions; + njs_object_t *o; + njs_object_prop_t *prop; + nxt_lvlhsh_each_t lhe; + + size = 0; + o = object; + + do { + nxt_lvlhsh_each_init(&lhe, &njs_object_hash_proto); + + for ( ;; ) { + prop = nxt_lvlhsh_each(&o->hash, &lhe); + if (prop == NULL) { + break; + } + + size++; + } + + nxt_lvlhsh_each_init(&lhe, &njs_object_hash_proto); + + for ( ;; ) { + prop = nxt_lvlhsh_each(&o->shared_hash, &lhe); + if (prop == NULL) { + break; + } + + size++; + } + + o = o->__proto__; + + } while (o != NULL); + + completions = nxt_array_create(size, sizeof(nxt_str_t), + &njs_array_mem_proto, vm->mem_cache_pool); + + if (nxt_slow_path(completions == NULL)) { + return NULL; + } + + n = 0; + o = object; + compl = completions->start; + + do { + nxt_lvlhsh_each_init(&lhe, &njs_object_hash_proto); + + for ( ;; ) { + prop = nxt_lvlhsh_each(&o->hash, &lhe); + if (prop == NULL) { + break; + } + + njs_string_get(&prop->name, &compl[n]); + + for (k = 0; k < n; k++) { + if (nxt_strstr_eq(&compl[k], &compl[n])) { + break; + } + } + + if (k == n) { + n++; + } + } + + nxt_lvlhsh_each_init(&lhe, &njs_object_hash_proto); + + for ( ;; ) { + prop = nxt_lvlhsh_each(&o->shared_hash, &lhe); + if (prop == NULL) { + break; + } + + njs_string_get(&prop->name, &compl[n]); + + for (k = 0; k < n; k++) { + if (nxt_strstr_eq(&compl[k], &compl[n])) { + break; + } + } + + if (k == n) { + n++; + } + } + + o = o->__proto__; + + } while (o != NULL); + + completions->items = n; + + return completions; +} + + nxt_int_t njs_builtin_match_native_function(njs_vm_t *vm, njs_function_t *function, nxt_str_t *name) diff -r 94c42736a730 -r d45f6f0079ab njs/njscript.h --- a/njs/njscript.h Tue Oct 03 21:24:58 2017 +0300 +++ b/njs/njscript.h Wed Oct 04 18:58:10 2017 +0300 @@ -123,7 +123,7 @@ NXT_EXPORT void *njs_value_data(njs_valu NXT_EXPORT nxt_int_t njs_value_string_copy(njs_vm_t *vm, nxt_str_t *retval, njs_value_t *value, uintptr_t *next); -NXT_EXPORT const char **njs_vm_completions(njs_vm_t *vm); +NXT_EXPORT nxt_array_t *njs_vm_completions(njs_vm_t *vm, nxt_str_t *expression); extern const nxt_mem_proto_t njs_vm_mem_cache_pool_proto; diff -r 94c42736a730 -r d45f6f0079ab njs/test/njs_expect_test.exp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/njs/test/njs_expect_test.exp Wed Oct 04 18:58:10 2017 +0300 @@ -0,0 +1,168 @@ +# +# Copyright (C) Dmitry Volyntsev +# Copyright (C) NGINX, Inc. +# + +proc njs_test {body} { + spawn -nottycopy njs + expect "interactive njscript\r +\r +v. -> the properties and prototype methods of v.\r +type console.help() for more information\r +\r +>> " + + set len [llength $body] + for {set i 0} {$i < $len} {incr i} { + set pair [lindex $body $i] + send [lindex $pair 0] + expect [lindex $pair 1] + } + + # Ctrl-C + send \x03 + expect eof +} + +# simple multi line interation +njs_test { + {"var a = 1\r\n" + "var a = 1\r\nundefined\r\n>> "} + {"a *= 2\r\n" + "a *= 2\r\n2\r\n>> "} +} + +# Global completions, no +njs_test { + {"\t\tn" + "\a\r\nDisplay all*possibilities? (y or n)*>> "} +} + +# Global completions, yes +njs_test { + {"\t\ty" + "\a\r\nDisplay all*possibilities? (y or n)*abstract"} +} + +# Global completions, single partial match +njs_test { + {"O\t" + "O\abject"} +} + +njs_test { + {"M\t" + "M\aath"} +} + +njs_test { + {"conso\t" + "conso\ale"} +} + +# Global completions, multiple partial match +njs_test { + {"cons\t\t" + "console*console.help*console.log*const"} +} + +njs_test { + {"O\t" + "O\abject"} + {".\t\t" + "Object.create*Object.isSealed"} +} + +njs_test { + {"Object.\t\t" + "Object.create*Object.isSealed"} +} + +njs_test { + {"Object.g\t" + "Object.g\aet"} + {"\t" + "Object.getOwnPropertyDescriptor*Object.getPrototypeOf"} +} + +njs_test { + {"M\t" + "M\aath"} + {".\t\t" + "Math.__proto__*Math.cbrt*Math.fround*Math.log2"} +} + +# Global completions, no matches +njs_test { + {"1.\t\t" + "1."} +} + +njs_test { + {"1..\t\t" + "1.."} +} + +njs_test { + {"'abc'.\t\t" + "'abc'."} +} + +# Global completions, global vars +njs_test { + {"var a = 1; var aa = 2\r\n" + "var a = 1; var aa = 2\r\nundefined\r\n>> "} + {"a\t\t" + "a*aa*abstract"} +} + +njs_test { + {"var zz = 1\r\n" + "var zz = 1\r\nundefined\r\n>> "} + {"1 + z\t\r\n" + "1 + zz\r\n2"} +} + +njs_test { + {"unknown_var\t\t" + "unknown_var"} +} + +njs_test { + {"unknown_var.\t\t" + "unknown_var."} +} + +# An object's level completions +njs_test { + {"var o = {zz:1, zb:2}\r\n" + "var o = {zz:1, zb:2}\r\nundefined\r\n>> "} + {"o.z\t\t" + "o.zb*o.zz"} +} + +njs_test { + {"var d = new Date()\r\n" + "var d = new Date()\r\nundefined\r\n>> "} + {"d.to\t\t" + "d.toDateString*d.toLocaleDateString*d.toString"} +} + +njs_test { + {"var o = {a:new Date()}\r\n" + "var o = {a:new Date()}\r\nundefined\r\n>> "} + {"o.a.to\t\t" + "o.a.toDateString*o.a.toLocaleDateString*o.a.toString"} +} + +# console object +njs_test { + {"console.log()\r\n" + "console.log()\r\n\r\nundefined\r\n>> "} + {"console.log(1)\r\n" + "console.log(1)\r\n1\r\nundefined\r\n>> "} + {"console.log('abc')\r\n" + "console.log('abc')\r\nabc\r\nundefined\r\n>> "} + {"console.help()\r\n" + "console.help()\r\nVM built-in objects:"} +} diff -r 94c42736a730 -r d45f6f0079ab nxt/auto/configure --- a/nxt/auto/configure Tue Oct 03 21:24:58 2017 +0300 +++ b/nxt/auto/configure Wed Oct 04 18:58:10 2017 +0300 @@ -56,3 +56,4 @@ END . ${NXT_AUTO}getrandom . ${NXT_AUTO}pcre . ${NXT_AUTO}editline +. ${NXT_AUTO}expect diff -r 94c42736a730 -r d45f6f0079ab nxt/auto/expect --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/nxt/auto/expect Wed Oct 04 18:58:10 2017 +0300 @@ -0,0 +1,31 @@ + +# Copyright (C) Dmitry Volyntsev +# Copyright (C) NGINX, Inc. + +nxt_found=no +$nxt_echo -n "checking for expect ..." + +if /bin/sh -c "(expect -v)" >> $NXT_AUTOCONF_ERR 2>&1; then + nxt_found=yes +fi + +if [ $nxt_found = yes ]; then + $nxt_echo " found" + $nxt_echo " + Expect version: `expect -v`" + cat << END >> $NXT_MAKEFILE_CONF + +njs_expect_test: njs njs/test/njs_expect_test.exp + PATH=\$(PATH):\$(NXT_BUILDDIR) expect -f njs/test/njs_expect_test.exp +END + +else + $nxt_echo " not found" + $nxt_echo " - expect tests are disabled" + + cat << END >> $NXT_MAKEFILE_CONF + +njs_expect_test: + @echo "Skipping expect tests" +END + +fi From vbart at nginx.com Wed Oct 4 18:15:41 2017 From: vbart at nginx.com (Valentin Bartenev) Date: Wed, 04 Oct 2017 18:15:41 +0000 Subject: [nginx] HTTP/2: enforce writing the sync request body buffer to file. Message-ID: details: http://hg.nginx.org/nginx/rev/b6dc472299da branches: changeset: 7118:b6dc472299da user: Valentin Bartenev date: Wed Oct 04 21:15:15 2017 +0300 description: HTTP/2: enforce writing the sync request body buffer to file. The sync flag of HTTP/2 request body buffer is used when the size of request body is unknown or bigger than configured "client_body_buffer_size". In this case the buffer points to body data inside the global receive buffer that is used for reading all HTTP/2 connections in the worker process. Thus, when the sync flag is set, the buffer must be flushed to a temporary file, otherwise the request body data can be overwritten. Previously, the sync buffer wasn't flushed to a temporary file if the whole body was received in one DATA frame with the END_STREAM flag and wasn't copied into the HTTP/2 body preread buffer. As a result, the request body might be corrupted (ticket #1384). Now, setting r->request_body_in_file_only enforces writing the sync buffer to a temporary file in all cases. diffstat: src/http/v2/ngx_http_v2.c | 7 ++----- 1 files changed, 2 insertions(+), 5 deletions(-) diffs (24 lines): diff -r dbd77a638eb7 -r b6dc472299da src/http/v2/ngx_http_v2.c --- a/src/http/v2/ngx_http_v2.c Tue Oct 03 18:19:27 2017 +0300 +++ b/src/http/v2/ngx_http_v2.c Wed Oct 04 21:15:15 2017 +0300 @@ -3589,11 +3589,6 @@ ngx_http_v2_read_request_body(ngx_http_r rb->buf = ngx_create_temp_buf(r->pool, (size_t) len); } else { - if (stream->preread) { - /* enforce writing preread buffer to file */ - r->request_body_in_file_only = 1; - } - rb->buf = ngx_calloc_buf(r->pool); if (rb->buf != NULL) { @@ -3694,6 +3689,8 @@ ngx_http_v2_process_request_body(ngx_htt buf->pos = buf->start = pos; buf->last = buf->end = pos + size; + r->request_body_in_file_only = 1; + } else { if (size > (size_t) (buf->end - buf->last)) { ngx_log_error(NGX_LOG_INFO, fc->log, 0, From mdounin at mdounin.ru Wed Oct 4 19:07:53 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Wed, 04 Oct 2017 19:07:53 +0000 Subject: [nginx] Fixed buffer overread with unix sockets after accept(). Message-ID: details: http://hg.nginx.org/nginx/rev/fef61d26da39 branches: changeset: 7119:fef61d26da39 user: Maxim Dounin date: Wed Oct 04 21:19:33 2017 +0300 description: Fixed buffer overread with unix sockets after accept(). Some OSes (notably macOS, NetBSD, and Solaris) allow unix socket addresses larger than struct sockaddr_un. Moreover, some of them (macOS, Solaris) return socklen of the socket address before it was truncated to fit the buffer provided. As such, on these systems socklen must not be used without additional check that it is within the buffer provided. Appropriate checks added to ngx_event_accept() (after accept()), ngx_event_recvmsg() (after recvmsg()), and ngx_set_inherited_sockets() (after getsockname()). We also obtain socket addresses via getsockname() in ngx_connection_local_sockaddr(), but it does not need any checks as it is only used for INET and INET6 sockets (as there can be no wildcard unix sockets). diffstat: src/core/ngx_connection.c | 4 ++++ src/event/ngx_event_accept.c | 8 ++++++++ 2 files changed, 12 insertions(+), 0 deletions(-) diffs (39 lines): diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -165,6 +165,10 @@ ngx_set_inherited_sockets(ngx_cycle_t *c continue; } + if (ls[i].socklen > (socklen_t) sizeof(ngx_sockaddr_t)) { + ls[i].socklen = sizeof(ngx_sockaddr_t); + } + switch (ls[i].sockaddr->sa_family) { #if (NGX_HAVE_INET6) diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c --- a/src/event/ngx_event_accept.c +++ b/src/event/ngx_event_accept.c @@ -164,6 +164,10 @@ ngx_event_accept(ngx_event_t *ev) return; } + if (socklen > (socklen_t) sizeof(ngx_sockaddr_t)) { + socklen = sizeof(ngx_sockaddr_t); + } + c->sockaddr = ngx_palloc(c->pool, socklen); if (c->sockaddr == NULL) { ngx_close_accepted_connection(c); @@ -440,6 +444,10 @@ ngx_event_recvmsg(ngx_event_t *ev) c->type = SOCK_DGRAM; c->socklen = msg.msg_namelen; + if (c->socklen > (socklen_t) sizeof(ngx_sockaddr_t)) { + c->socklen = sizeof(ngx_sockaddr_t); + } + #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_active, 1); #endif From mdounin at mdounin.ru Wed Oct 4 19:07:55 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Wed, 04 Oct 2017 19:07:55 +0000 Subject: [nginx] Fixed handling of non-null-terminated unix sockets. Message-ID: details: http://hg.nginx.org/nginx/rev/874171c3c71a branches: changeset: 7120:874171c3c71a user: Maxim Dounin date: Wed Oct 04 21:19:38 2017 +0300 description: Fixed handling of non-null-terminated unix sockets. At least FreeBSD, macOS, NetBSD, and OpenBSD can return unix sockets with non-null-terminated sun_path. Additionally, the address may become non-null-terminated if it does not fit into the buffer provided and was truncated (may happen on macOS, NetBSD, and Solaris, which allow unix socket addresess larger than struct sockaddr_un). As such, ngx_sock_ntop() might overread the sockaddr provided, as it used "%s" format and thus assumed null-terminated string. To fix this, the ngx_strnlen() function was introduced, and it is now used to calculate correct length of sun_path. diffstat: src/core/ngx_inet.c | 4 +++- src/core/ngx_string.c | 16 ++++++++++++++++ src/core/ngx_string.h | 2 ++ 3 files changed, 21 insertions(+), 1 deletions(-) diffs (52 lines): diff --git a/src/core/ngx_inet.c b/src/core/ngx_inet.c --- a/src/core/ngx_inet.c +++ b/src/core/ngx_inet.c @@ -241,7 +241,9 @@ ngx_sock_ntop(struct sockaddr *sa, sockl p = ngx_snprintf(text, len, "unix:%Z"); } else { - p = ngx_snprintf(text, len, "unix:%s%Z", saun->sun_path); + n = ngx_strnlen((u_char *) saun->sun_path, + socklen - offsetof(struct sockaddr_un, sun_path)); + p = ngx_snprintf(text, len, "unix:%*s%Z", n, saun->sun_path); } /* we do not include trailing zero in address length */ diff --git a/src/core/ngx_string.c b/src/core/ngx_string.c --- a/src/core/ngx_string.c +++ b/src/core/ngx_string.c @@ -29,6 +29,22 @@ ngx_strlow(u_char *dst, u_char *src, siz } +size_t +ngx_strnlen(u_char *p, size_t n) +{ + size_t i; + + for (i = 0; i < n; i++) { + + if (p[i] == '\0') { + return i; + } + } + + return n; +} + + u_char * ngx_cpystrn(u_char *dst, u_char *src, size_t n) { diff --git a/src/core/ngx_string.h b/src/core/ngx_string.h --- a/src/core/ngx_string.h +++ b/src/core/ngx_string.h @@ -60,6 +60,8 @@ void ngx_strlow(u_char *dst, u_char *src #define ngx_strstr(s1, s2) strstr((const char *) s1, (const char *) s2) #define ngx_strlen(s) strlen((const char *) s) +size_t ngx_strnlen(u_char *p, size_t n); + #define ngx_strchr(s1, c) strchr((const char *) s1, (int) c) static ngx_inline u_char * From mdounin at mdounin.ru Wed Oct 4 19:07:58 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Wed, 04 Oct 2017 19:07:58 +0000 Subject: [nginx] Fixed handling of unix sockets in $binary_remote_addr. Message-ID: details: http://hg.nginx.org/nginx/rev/924b6ef942bf branches: changeset: 7121:924b6ef942bf user: Maxim Dounin date: Wed Oct 04 21:19:42 2017 +0300 description: Fixed handling of unix sockets in $binary_remote_addr. Previously, unix sockets were treated as AF_INET ones, and this may result in buffer overread on Linux, where unbound unix sockets have 2-byte addresses. Note that it is not correct to use just sun_path as a binary representation for unix sockets. This will result in an empty string for unbound unix sockets, and thus behaviour of limit_req and limit_conn will change when switching from $remote_addr to $binary_remote_addr. As such, normal text representation is used. Reported by Stephan Dollberg. diffstat: src/http/ngx_http_variables.c | 12 ++++++++++++ src/stream/ngx_stream_variables.c | 12 ++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-) diffs (44 lines): diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c --- a/src/http/ngx_http_variables.c +++ b/src/http/ngx_http_variables.c @@ -1240,6 +1240,18 @@ ngx_http_variable_binary_remote_addr(ngx break; #endif +#if (NGX_HAVE_UNIX_DOMAIN) + case AF_UNIX: + + v->len = r->connection->addr_text.len; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = r->connection->addr_text.data; + + break; +#endif + default: /* AF_INET */ sin = (struct sockaddr_in *) r->connection->sockaddr; diff --git a/src/stream/ngx_stream_variables.c b/src/stream/ngx_stream_variables.c --- a/src/stream/ngx_stream_variables.c +++ b/src/stream/ngx_stream_variables.c @@ -481,6 +481,18 @@ ngx_stream_variable_binary_remote_addr(n break; #endif +#if (NGX_HAVE_UNIX_DOMAIN) + case AF_UNIX: + + v->len = s->connection->addr_text.len; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = s->connection->addr_text.data; + + break; +#endif + default: /* AF_INET */ sin = (struct sockaddr_in *) s->connection->sockaddr; From mdounin at mdounin.ru Wed Oct 4 19:11:42 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Wed, 4 Oct 2017 22:11:42 +0300 Subject: Heap buffer overflow (read) when using $binary_remote_addr with unix sockets In-Reply-To: <3A98A020-EE7F-428A-BFF9-11FF545BFD70@nginx.com> References: <3A98A020-EE7F-428A-BFF9-11FF545BFD70@nginx.com> Message-ID: <20171004191142.GX16067@mdounin.ru> Hello! On Wed, Sep 20, 2017 at 04:21:22PM +0300, Sergey Kandaurov wrote: > > On 15 Aug 2017, at 13:10, Stephan Dollberg via nginx-devel wrote: > > > > Hi, > > > > When using $binary_remote_addr together with unix sockets (without > > using X-Real-Ip) there is a heap buffer overread of two bytes. > > > > The problem is that we only allocate two bytes for c->sockaddr here > > http://hg.nginx.org/nginx/file/tip/src/event/ngx_event_accept.c#l167 > > but later on assume it to be of size four > > http://hg.nginx.org/nginx/file/tip/src/http/ngx_http_variables.c#l1246 > > > > > > Thanks, this is a valid report. > The reason is that UNIX-domain sockets support > is not implemented for $binary_remote_addr. > There are actually more issues, we are working on it. Fixes for this and related issues committed: http://hg.nginx.org/nginx/rev/fef61d26da39 http://hg.nginx.org/nginx/rev/874171c3c71a http://hg.nginx.org/nginx/rev/924b6ef942bf Thanks. -- Maxim Dounin http://nginx.org/ From xeioex at nginx.com Thu Oct 5 12:51:00 2017 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Oct 2017 12:51:00 +0000 Subject: [njs] Fixed default makefile target is libedit is unavailable. Message-ID: details: http://hg.nginx.org/njs/rev/d0e24fa20991 branches: changeset: 412:d0e24fa20991 user: Dmitry Volyntsev date: Thu Oct 05 15:50:16 2017 +0300 description: Fixed default makefile target is libedit is unavailable. Previously, if libedit was unavailable, any extra target which could be added by feature scripts became the first and default target. It breaks builds without arguments. Adding the default target in this case to ensure that the proper target is always selected. diffstat: nxt/auto/editline | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (13 lines): diff -r d45f6f0079ab -r d0e24fa20991 nxt/auto/editline --- a/nxt/auto/editline Wed Oct 04 18:58:10 2017 +0300 +++ b/nxt/auto/editline Thu Oct 05 15:50:16 2017 +0300 @@ -39,4 +39,9 @@ END else $nxt_echo " - building interactive shell is not possible" + cat << END >> $NXT_MAKEFILE_CONF + +default: all +END + fi From xeioex at nginx.com Thu Oct 5 12:51:02 2017 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Oct 2017 12:51:02 +0000 Subject: [njs] Skipping expect tests if libedit is not available. Message-ID: details: http://hg.nginx.org/njs/rev/6789d9142b05 branches: changeset: 413:6789d9142b05 user: Dmitry Volyntsev date: Thu Oct 05 15:50:17 2017 +0300 description: Skipping expect tests if libedit is not available. diffstat: nxt/auto/editline | 2 ++ nxt/auto/expect | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletions(-) diffs (43 lines): diff -r d0e24fa20991 -r 6789d9142b05 nxt/auto/editline --- a/nxt/auto/editline Thu Oct 05 15:50:16 2017 +0300 +++ b/nxt/auto/editline Thu Oct 05 15:50:17 2017 +0300 @@ -28,6 +28,7 @@ if [ $nxt_found = no ]; then fi if [ $nxt_found = yes ]; then + NXT_HAVE_LIBEDIT=YES cat << END >> $NXT_MAKEFILE_CONF NXT_EDITLINE_CFLAGS = $nxt_feature_incs @@ -37,6 +38,7 @@ default: njs END else + NXT_HAVE_LIBEDIT=NO $nxt_echo " - building interactive shell is not possible" cat << END >> $NXT_MAKEFILE_CONF diff -r d0e24fa20991 -r 6789d9142b05 nxt/auto/expect --- a/nxt/auto/expect Thu Oct 05 15:50:16 2017 +0300 +++ b/nxt/auto/expect Thu Oct 05 15:50:17 2017 +0300 @@ -12,6 +12,12 @@ fi if [ $nxt_found = yes ]; then $nxt_echo " found" $nxt_echo " + Expect version: `expect -v`" + +else + $nxt_echo " not found" +fi + +if [ $nxt_found = yes -a $NXT_HAVE_LIBEDIT = YES ]; then cat << END >> $NXT_MAKEFILE_CONF njs_expect_test: njs njs/test/njs_expect_test.exp @@ -19,7 +25,6 @@ njs_expect_test: njs njs/test/njs_expect END else - $nxt_echo " not found" $nxt_echo " - expect tests are disabled" cat << END >> $NXT_MAKEFILE_CONF From xeioex at nginx.com Thu Oct 5 12:51:03 2017 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Oct 2017 12:51:03 +0000 Subject: [njs] Fixed various dead store assignments. Message-ID: details: http://hg.nginx.org/njs/rev/b4462c7dc1b1 branches: changeset: 414:b4462c7dc1b1 user: Dmitry Volyntsev date: Thu Oct 05 15:50:36 2017 +0300 description: Fixed various dead store assignments. diffstat: njs/njs_json.c | 2 +- njs/njs_parser_expression.c | 6 ------ 2 files changed, 1 insertions(+), 7 deletions(-) diffs (42 lines): diff -r 6789d9142b05 -r b4462c7dc1b1 njs/njs_json.c --- a/njs/njs_json.c Thu Oct 05 15:50:17 2017 +0300 +++ b/njs/njs_json.c Thu Oct 05 15:50:36 2017 +0300 @@ -1014,7 +1014,7 @@ njs_json_parse_continuation(njs_vm_t *vm } } else { - state = njs_json_pop_parse_state(parse); + (void) njs_json_pop_parse_state(parse); } return njs_json_parse_continuation_apply(vm, parse); diff -r 6789d9142b05 -r b4462c7dc1b1 njs/njs_parser_expression.c --- a/njs/njs_parser_expression.c Thu Oct 05 15:50:17 2017 +0300 +++ b/njs/njs_parser_expression.c Thu Oct 05 15:50:36 2017 +0300 @@ -269,8 +269,6 @@ njs_parser_var_expression(njs_vm_t *vm, } for ( ;; ) { - size = sizeof(njs_vmcode_3addr_t); - switch (token) { case NJS_TOKEN_ASSIGNMENT: @@ -295,8 +293,6 @@ njs_parser_var_expression(njs_vm_t *vm, return token; } - node = parser->node; - if (!njs_parser_is_lvalue(parser->node)) { nxt_alert(&vm->trace, NXT_LEVEL_ERROR, "ReferenceError: Invalid left-hand side in assignment"); @@ -435,8 +431,6 @@ njs_parser_assignment_expression(njs_vm_ return token; } - node = parser->node; - if (!njs_parser_is_lvalue(parser->node)) { nxt_alert(&vm->trace, NXT_LEVEL_ERROR, "ReferenceError: Invalid left-hand side in assignment"); From jan.prachar at gmail.com Thu Oct 5 13:12:42 2017 From: jan.prachar at gmail.com (Jan =?UTF-8?Q?Pracha=C5=99?=) Date: Thu, 05 Oct 2017 15:12:42 +0200 Subject: [PATCH] Proxy: tcp fast open in client mode Message-ID: <1507209162.12229.4.camel@gmail.com> What do you think about adding support for tcp fast open in the proxy module when connecting to upstream? --- nginx/conf/nginx/nginx.conf | 1 + nginx/contrib/vim/syntax/nginx.vim | 1 + nginx/src/core/ngx_connection.h | 1 + nginx/src/event/ngx_event_connect.c | 9 +++++++++ nginx/src/event/ngx_event_connect.h | 1 + nginx/src/http/modules/ngx_http_proxy_module.c | 15 +++++++++++++++ nginx/src/http/ngx_http_upstream.h | 1 + nginx/src/os/unix/ngx_send.c | 12 ++++++++++-- nginx/src/os/unix/ngx_writev_chain.c | 14 ++++++++++++++ 9 files changed, 53 insertions(+), 2 deletions(-) diff --git a/nginx/conf/nginx/nginx.conf b/nginx/conf/nginx/nginx.conf index e1a090132..6d32dfac5 100644 --- a/nginx/conf/nginx/nginx.conf +++ b/nginx/conf/nginx/nginx.conf @@ -107,6 +107,7 @@ http proxy_max_temp_file_size 0; proxy_ignore_client_abort on; proxy_temp_path /var/cache/nginx/tmp; + proxy_tcp_fastopen on; proxy_set_header Host $resource_origin; proxy_set_header Via $upstream_header_via; diff --git a/nginx/contrib/vim/syntax/nginx.vim b/nginx/contrib/vim/syntax/nginx.vim index e7aadea9c..63e05b3bf 100644 --- a/nginx/contrib/vim/syntax/nginx.vim +++ b/nginx/contrib/vim/syntax/nginx.vim @@ -399,6 +399,7 @@ syn keyword ngxDirective contained proxy_headers_hash_bucket_size syn keyword ngxDirective contained proxy_headers_hash_max_size syn keyword ngxDirective contained proxy_hide_header syn keyword ngxDirective contained proxy_http_version +syn keyword ngxDirective contained proxy_tcp_fastopen syn keyword ngxDirective contained proxy_ignore_client_abort syn keyword ngxDirective contained proxy_ignore_headers syn keyword ngxDirective contained proxy_intercept_errors diff --git a/nginx/src/core/ngx_connection.h b/nginx/src/core/ngx_connection.h index e4dfe5879..bcfe2f6d8 100644 --- a/nginx/src/core/ngx_connection.h +++ b/nginx/src/core/ngx_connection.h @@ -179,6 +179,7 @@ struct ngx_connection_s { unsigned sndlowat:1; unsigned tcp_nodelay:2; /* ngx_connection_tcp_nodelay_e */ unsigned tcp_nopush:2; /* ngx_connection_tcp_nopush_e */ + unsigned tcp_fastopen:1; unsigned need_last_buf:1; diff --git a/nginx/src/event/ngx_event_connect.c b/nginx/src/event/ngx_event_connect.c index c5bb80681..f24804dd9 100644 --- a/nginx/src/event/ngx_event_connect.c +++ b/nginx/src/event/ngx_event_connect.c @@ -190,6 +190,15 @@ ngx_event_connect_peer(ngx_peer_connection_t *pc) ngx_log_debug3(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connect to %V, fd:%d #%uA", pc->name, s, c- >number); +#if (NGX_LINUX) && defined(MSG_FASTOPEN) + if (pc->tcp_fastopen && type == SOCK_STREAM) { + rc = 0; /* Do nothing now */ + + c->sockaddr = pc->sockaddr; + c->socklen = pc->socklen; + c->tcp_fastopen = 1; + } else +#endif rc = connect(s, pc->sockaddr, pc->socklen); if (rc == -1) { diff --git a/nginx/src/event/ngx_event_connect.h b/nginx/src/event/ngx_event_connect.h index 72d21d7f3..ebc6b78a7 100644 --- a/nginx/src/event/ngx_event_connect.h +++ b/nginx/src/event/ngx_event_connect.h @@ -62,6 +62,7 @@ struct ngx_peer_connection_s { unsigned cached:1; unsigned transparent:1; + unsigned tcp_fastopen:1; /* ngx_connection_log_error_e */ unsigned log_error:2; diff --git a/nginx/src/http/modules/ngx_http_proxy_module.c b/nginx/src/http/modules/ngx_http_proxy_module.c index d8b5541e1..27c25bdfb 100644 --- a/nginx/src/http/modules/ngx_http_proxy_module.c +++ b/nginx/src/http/modules/ngx_http_proxy_module.c @@ -653,6 +653,13 @@ static ngx_command_t ngx_http_proxy_commands[] = { offsetof(ngx_http_proxy_loc_conf_t, http_version), &ngx_http_proxy_http_version }, + { ngx_string("proxy_tcp_fastopen"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_ FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, upstream.tcp_fastopen), + NULL }, + #if (NGX_HTTP_SSL) { ngx_string("proxy_ssl_session_reuse"), @@ -956,6 +963,10 @@ ngx_http_proxy_handler(ngx_http_request_t *r) r->request_body_no_buffering = 1; } + if (plcf->upstream.tcp_fastopen) { + u->peer.tcp_fastopen = 1; + } + rc = ngx_http_read_client_request_body(r, ngx_http_upstream_init); if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { @@ -3074,6 +3085,8 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf) conf->http_version = NGX_CONF_UNSET_UINT; + conf->upstream.tcp_fastopen = NGX_CONF_UNSET; + conf->headers_hash_max_size = NGX_CONF_UNSET_UINT; conf->headers_hash_bucket_size = NGX_CONF_UNSET_UINT; @@ -3473,6 +3486,8 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_uint_value(conf->http_version, prev->http_version, NGX_HTTP_VERSION_10); + ngx_conf_merge_value(conf->upstream.tcp_fastopen, prev- >upstream.tcp_fastopen, 0); + ngx_conf_merge_uint_value(conf->headers_hash_max_size, prev->headers_hash_max_size, 512); diff --git a/nginx/src/http/ngx_http_upstream.h b/nginx/src/http/ngx_http_upstream.h index 089374f65..b2a360f86 100644 --- a/nginx/src/http/ngx_http_upstream.h +++ b/nginx/src/http/ngx_http_upstream.h @@ -184,6 +184,7 @@ typedef struct { ngx_flag_t intercept_errors; ngx_flag_t cyclic_temp_file; ngx_flag_t force_ranges; + ngx_flag_t tcp_fastopen; ngx_path_t *temp_path; diff --git a/nginx/src/os/unix/ngx_send.c b/nginx/src/os/unix/ngx_send.c index 61ea20252..0559cb76d 100644 --- a/nginx/src/os/unix/ngx_send.c +++ b/nginx/src/os/unix/ngx_send.c @@ -31,6 +31,14 @@ ngx_unix_send(ngx_connection_t *c, u_char *buf, size_t size) #endif for ( ;; ) { +#if (NGX_LINUX) && defined(MSG_FASTOPEN) + if (c->tcp_fastopen) { + + n = sendto(c->fd, buf, size, MSG_FASTOPEN, c->sockaddr, c- >socklen); + c->tcp_fastopen = 0; + + } else +#endif n = send(c->fd, buf, size, 0); ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -54,13 +62,13 @@ ngx_unix_send(ngx_connection_t *c, u_char *buf, size_t size) return n; } - if (err == NGX_EAGAIN || err == NGX_EINTR) { + if (err == NGX_EAGAIN || err == NGX_EINTR || err == NGX_EINPROGRESS) { wev->ready = 0; ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err, "send() not ready"); - if (err == NGX_EAGAIN) { + if (err == NGX_EAGAIN || err == NGX_EINPROGRESS) { return NGX_AGAIN; } diff --git a/nginx/src/os/unix/ngx_writev_chain.c b/nginx/src/os/unix/ngx_writev_chain.c index e38a3aae0..349b07900 100644 --- a/nginx/src/os/unix/ngx_writev_chain.c +++ b/nginx/src/os/unix/ngx_writev_chain.c @@ -186,6 +186,19 @@ ngx_writev(ngx_connection_t *c, ngx_iovec_t *vec) eintr: +#if (NGX_LINUX) && defined(MSG_FASTOPEN) + if (c->tcp_fastopen) { + if (vec->count > 0) { + + n = sendto(c->fd, vec->iovs[0].iov_base, vec- >iovs[0].iov_len, + MSG_FASTOPEN, c->sockaddr, c->socklen); + c->tcp_fastopen = 0; + + } else { + n = 0; + } + } else +#endif n = writev(c->fd, vec->iovs, vec->count); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -195,6 +208,7 @@ eintr: err = ngx_errno; switch (err) { + case NGX_EINPROGRESS: case NGX_EAGAIN: ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err, "writev() not ready"); -- 2.14.2 From mdounin at mdounin.ru Thu Oct 5 14:25:44 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 05 Oct 2017 14:25:44 +0000 Subject: [nginx] Fixed build without IPv6, broken by 874171c3c71a. Message-ID: details: http://hg.nginx.org/nginx/rev/935b1902a7dd branches: changeset: 7122:935b1902a7dd user: Maxim Dounin date: Thu Oct 05 16:50:35 2017 +0300 description: Fixed build without IPv6, broken by 874171c3c71a. diffstat: src/core/ngx_inet.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (16 lines): diff --git a/src/core/ngx_inet.c b/src/core/ngx_inet.c --- a/src/core/ngx_inet.c +++ b/src/core/ngx_inet.c @@ -182,9 +182,11 @@ ngx_sock_ntop(struct sockaddr *sa, sockl ngx_uint_t port) { u_char *p; +#if (NGX_HAVE_INET6 || NGX_HAVE_UNIX_DOMAIN) + size_t n; +#endif struct sockaddr_in *sin; #if (NGX_HAVE_INET6) - size_t n; struct sockaddr_in6 *sin6; #endif #if (NGX_HAVE_UNIX_DOMAIN) From xeioex at nginx.com Thu Oct 5 15:30:51 2017 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Oct 2017 15:30:51 +0000 Subject: [njs] Making build and test targets separate. Message-ID: details: http://hg.nginx.org/njs/rev/65607378a01b branches: changeset: 415:65607378a01b user: Dmitry Volyntsev date: Thu Oct 05 18:22:56 2017 +0300 description: Making build and test targets separate. Previously, if libedit was unavailable the default action was to build libnjs and run tests. However, if it was available the default action was just building binaries. This patch makes the two high-level targets which are intended to be invoked externally. The default one for building all available binaries and the test target for running all available tests. diffstat: Makefile | 4 +++- nxt/auto/editline | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diffs (26 lines): diff -r b4462c7dc1b1 -r 65607378a01b Makefile --- a/Makefile Thu Oct 05 15:50:36 2017 +0300 +++ b/Makefile Thu Oct 05 18:22:56 2017 +0300 @@ -76,7 +76,9 @@ NXT_BUILDDIR = build all: test lib_test -njs: $(NXT_BUILDDIR)/njs +njs: $(NXT_BUILDDIR)/njs + +libnjs: $(NXT_BUILDDIR)/libnjs.a njs_interactive_test: njs_expect_test $(NXT_BUILDDIR)/njs_interactive_test $(NXT_BUILDDIR)/njs_interactive_test diff -r b4462c7dc1b1 -r 65607378a01b nxt/auto/editline --- a/nxt/auto/editline Thu Oct 05 15:50:36 2017 +0300 +++ b/nxt/auto/editline Thu Oct 05 18:22:56 2017 +0300 @@ -43,7 +43,7 @@ else cat << END >> $NXT_MAKEFILE_CONF -default: all +default: libnjs END fi From xeioex at nginx.com Thu Oct 5 15:30:53 2017 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Thu, 05 Oct 2017 15:30:53 +0000 Subject: [njs] Improved editline detection on NetBSD. Message-ID: details: http://hg.nginx.org/njs/rev/d6ab83febe2b branches: changeset: 416:d6ab83febe2b user: Dmitry Volyntsev date: Thu Oct 05 18:23:22 2017 +0300 description: Improved editline detection on NetBSD. diffstat: nxt/auto/editline | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diffs (20 lines): diff -r 65607378a01b -r d6ab83febe2b nxt/auto/editline --- a/nxt/auto/editline Thu Oct 05 18:22:56 2017 +0300 +++ b/nxt/auto/editline Thu Oct 05 18:23:22 2017 +0300 @@ -27,6 +27,16 @@ if [ $nxt_found = no ]; then . auto/feature fi +if [ $nxt_found = no ]; then + + # NetBSD + + nxt_feature="editline in /usr/include" + nxt_feature_incs="-I/usr/include/readline" + + . auto/feature +fi + if [ $nxt_found = yes ]; then NXT_HAVE_LIBEDIT=YES cat << END >> $NXT_MAKEFILE_CONF From mdounin at mdounin.ru Thu Oct 5 15:37:53 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 05 Oct 2017 15:37:53 +0000 Subject: [nginx] Upstream hash: limited number of tries in consistent case. Message-ID: details: http://hg.nginx.org/nginx/rev/6c52b24fcf8e branches: changeset: 7123:6c52b24fcf8e user: Maxim Dounin date: Thu Oct 05 17:42:59 2017 +0300 description: Upstream hash: limited number of tries in consistent case. While this may result in non-ideal distribution of requests if nginx won't be able to select a server in a reasonable number of attempts, this still looks better than severe performance degradation observed if there is no limit and there are many points configured (ticket #1030). This is also in line with what we do for other hash balancing methods. diffstat: src/http/modules/ngx_http_upstream_hash_module.c | 10 +++++++--- src/stream/ngx_stream_upstream_hash_module.c | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diffs (56 lines): diff --git a/src/http/modules/ngx_http_upstream_hash_module.c b/src/http/modules/ngx_http_upstream_hash_module.c --- a/src/http/modules/ngx_http_upstream_hash_module.c +++ b/src/http/modules/ngx_http_upstream_hash_module.c @@ -503,6 +503,11 @@ ngx_http_upstream_get_chash_peer(ngx_pee ngx_http_upstream_rr_peers_wlock(hp->rrp.peers); + if (hp->tries > 20 || hp->rrp.peers->single) { + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); + return hp->get_rr_peer(pc, &hp->rrp); + } + pc->cached = 0; pc->connection = NULL; @@ -577,10 +582,9 @@ ngx_http_upstream_get_chash_peer(ngx_pee hp->hash++; hp->tries++; - if (hp->tries >= points->number) { - pc->name = hp->rrp.peers->name; + if (hp->tries > 20) { ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); - return NGX_BUSY; + return hp->get_rr_peer(pc, &hp->rrp); } } diff --git a/src/stream/ngx_stream_upstream_hash_module.c b/src/stream/ngx_stream_upstream_hash_module.c --- a/src/stream/ngx_stream_upstream_hash_module.c +++ b/src/stream/ngx_stream_upstream_hash_module.c @@ -505,6 +505,11 @@ ngx_stream_upstream_get_chash_peer(ngx_p ngx_stream_upstream_rr_peers_wlock(hp->rrp.peers); + if (hp->tries > 20 || hp->rrp.peers->single) { + ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers); + return hp->get_rr_peer(pc, &hp->rrp); + } + pc->connection = NULL; now = ngx_time(); @@ -578,10 +583,9 @@ ngx_stream_upstream_get_chash_peer(ngx_p hp->hash++; hp->tries++; - if (hp->tries >= points->number) { - pc->name = hp->rrp.peers->name; + if (hp->tries > 20) { ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers); - return NGX_BUSY; + return hp->get_rr_peer(pc, &hp->rrp); } } From mdounin at mdounin.ru Thu Oct 5 15:37:56 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 05 Oct 2017 15:37:56 +0000 Subject: [nginx] Upstream hash: reordered peer checks. Message-ID: details: http://hg.nginx.org/nginx/rev/3b4fa572d56d branches: changeset: 7124:3b4fa572d56d user: Maxim Dounin date: Thu Oct 05 17:43:05 2017 +0300 description: Upstream hash: reordered peer checks. This slightly reduces cost of selecting a peer if all or almost all peers failed, see ticket #1030. There should be no measureable difference with other workloads. diffstat: src/http/modules/ngx_http_upstream_hash_module.c | 14 +++++++------- src/stream/ngx_stream_upstream_hash_module.c | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diffs (62 lines): diff --git a/src/http/modules/ngx_http_upstream_hash_module.c b/src/http/modules/ngx_http_upstream_hash_module.c --- a/src/http/modules/ngx_http_upstream_hash_module.c +++ b/src/http/modules/ngx_http_upstream_hash_module.c @@ -543,13 +543,6 @@ ngx_http_upstream_get_chash_peer(ngx_pee continue; } - if (peer->server.len != server->len - || ngx_strncmp(peer->server.data, server->data, server->len) - != 0) - { - continue; - } - if (peer->max_fails && peer->fails >= peer->max_fails && now - peer->checked <= peer->fail_timeout) @@ -561,6 +554,13 @@ ngx_http_upstream_get_chash_peer(ngx_pee continue; } + if (peer->server.len != server->len + || ngx_strncmp(peer->server.data, server->data, server->len) + != 0) + { + continue; + } + peer->current_weight += peer->effective_weight; total += peer->effective_weight; diff --git a/src/stream/ngx_stream_upstream_hash_module.c b/src/stream/ngx_stream_upstream_hash_module.c --- a/src/stream/ngx_stream_upstream_hash_module.c +++ b/src/stream/ngx_stream_upstream_hash_module.c @@ -544,13 +544,6 @@ ngx_stream_upstream_get_chash_peer(ngx_p continue; } - if (peer->server.len != server->len - || ngx_strncmp(peer->server.data, server->data, server->len) - != 0) - { - continue; - } - if (peer->max_fails && peer->fails >= peer->max_fails && now - peer->checked <= peer->fail_timeout) @@ -562,6 +555,13 @@ ngx_stream_upstream_get_chash_peer(ngx_p continue; } + if (peer->server.len != server->len + || ngx_strncmp(peer->server.data, server->data, server->len) + != 0) + { + continue; + } + peer->current_weight += peer->effective_weight; total += peer->effective_weight; From mdounin at mdounin.ru Thu Oct 5 16:03:15 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 5 Oct 2017 19:03:15 +0300 Subject: [PATCH] Proxy: tcp fast open in client mode In-Reply-To: <1507209162.12229.4.camel@gmail.com> References: <1507209162.12229.4.camel@gmail.com> Message-ID: <20171005160315.GD16067@mdounin.ru> Hello! On Thu, Oct 05, 2017 at 03:12:42PM +0200, Jan Pracha? wrote: > What do you think about adding support for tcp fast open in the proxy > module when connecting to upstream? I'm somewhat sceptical. It seems to be limited to very few use cases, and also there are lot of corner cases to consider. In particular, as far as I see the patch in question will break miserably when using SSL (not to mention it won't work on non-linux systems with TCP_FASTOPEN). It might be much better idea to use persistent connections to upstreams instead. -- Maxim Dounin http://nginx.org/ From mdounin at mdounin.ru Mon Oct 9 13:57:57 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 09 Oct 2017 13:57:57 +0000 Subject: [nginx] Upstream: even better handling of invalid headers in cache files. Message-ID: details: http://hg.nginx.org/nginx/rev/b8fc701daf8e branches: changeset: 7125:b8fc701daf8e user: Maxim Dounin date: Mon Oct 09 15:59:10 2017 +0300 description: Upstream: even better handling of invalid headers in cache files. When parsing of headers in a cache file fails, already parsed headers need to be cleared, and protocol state needs to be reinitialized. To do so, u->request_sent is now set to ensure ngx_http_upstream_reinit() will be called. This change complements improvements in 46ddff109e72. diffstat: src/http/ngx_http_upstream.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -584,6 +584,7 @@ ngx_http_upstream_init_request(ngx_http_ r->cached = 0; u->buffer.start = NULL; u->cache_status = NGX_HTTP_CACHE_MISS; + u->request_sent = 1; } if (ngx_http_upstream_cache_background_update(r, u) != NGX_OK) { From xeioex at nginx.com Mon Oct 9 17:38:41 2017 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Mon, 09 Oct 2017 17:38:41 +0000 Subject: [njs] Version 0.1.14. Message-ID: details: http://hg.nginx.org/njs/rev/d89d06dc638e branches: changeset: 417:d89d06dc638e user: Dmitry Volyntsev date: Mon Oct 09 20:32:21 2017 +0300 description: Version 0.1.14. diffstat: CHANGES | 11 +++++++++++ Makefile | 2 +- 2 files changed, 12 insertions(+), 1 deletions(-) diffs (28 lines): diff -r d6ab83febe2b -r d89d06dc638e CHANGES --- a/CHANGES Thu Oct 05 18:23:22 2017 +0300 +++ b/CHANGES Mon Oct 09 20:32:21 2017 +0300 @@ -1,3 +1,14 @@ + +Changes with nJScript 0.1.14 09 Oct 2017 + + *) Feature: JSON object. + + *) Feature: object level completions in interactive shell. + + *) Feature: various configure improvements. + + *) Bugfix: miscellaneous bugs have been fixed in the core and + interactive shell. Changes with nJScript 0.1.13 31 Aug 2017 diff -r d6ab83febe2b -r d89d06dc638e Makefile --- a/Makefile Thu Oct 05 18:23:22 2017 +0300 +++ b/Makefile Mon Oct 09 20:32:21 2017 +0300 @@ -1,5 +1,5 @@ -NJS_VER = 0.1.13 +NJS_VER = 0.1.14 NXT_LIB = nxt From xeioex at nginx.com Mon Oct 9 17:38:43 2017 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Mon, 09 Oct 2017 17:38:43 +0000 Subject: [njs] Added tag 0.1.14 for changeset d89d06dc638e Message-ID: details: http://hg.nginx.org/njs/rev/06aadeb164a3 branches: changeset: 418:06aadeb164a3 user: Dmitry Volyntsev date: Mon Oct 09 20:37:02 2017 +0300 description: Added tag 0.1.14 for changeset d89d06dc638e diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r d89d06dc638e -r 06aadeb164a3 .hgtags --- a/.hgtags Mon Oct 09 20:32:21 2017 +0300 +++ b/.hgtags Mon Oct 09 20:37:02 2017 +0300 @@ -12,3 +12,4 @@ b1456ef3e002376d9d146a8a02acf6a4a21748e9 fc5df33f4e6b02a673daf3728ff690fb1e09b95e 0.1.11 c07b060396be3622ca97b037a86076b61b850847 0.1.12 d548b78eb881ca799aa6fc8ba459d076f7db5ac8 0.1.13 +d89d06dc638e78f8635c0bfbcd02469ac1a08748 0.1.14 From leon at darkk.net.ru Mon Oct 9 17:44:07 2017 From: leon at darkk.net.ru (Leonid Evdokimov) Date: Mon, 9 Oct 2017 20:44:07 +0300 Subject: [patch]: document SHA-2 support in glibc crypt() In-Reply-To: <20171009165816.GA5388@darkk-ya-laptop> References: <20171009165402.GB5231@darkk-ya-laptop> <20171009165816.GA5388@darkk-ya-laptop> Message-ID: <20171009174407.GB5549@darkk-ya-laptop> Hello! I'd like to see few more lines about SHA-2 support in glibc crypt() in documentation. I've created small patch for nginx.org repo, it's attached. -- WBRBW, Leonid Evdokimov, xmpp:leon at darkk.net.ru http://darkk.net.ru tel:+79816800702 PGP: 6691 DE6B 4CCD C1C1 76A0 0D4A E1F2 A980 7F50 FAB2 -------------- next part -------------- A non-text attachment was scrubbed... Name: crypt-sha2-doc.diff Type: text/x-diff Size: 2530 bytes Desc: not available URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 819 bytes Desc: not available URL: From mdounin at mdounin.ru Mon Oct 9 19:44:11 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 9 Oct 2017 22:44:11 +0300 Subject: [patch]: document SHA-2 support in glibc crypt() In-Reply-To: <20171009174407.GB5549@darkk-ya-laptop> References: <20171009165402.GB5231@darkk-ya-laptop> <20171009165816.GA5388@darkk-ya-laptop> <20171009174407.GB5549@darkk-ya-laptop> Message-ID: <20171009194411.GH75166@mdounin.ru> Hello! On Mon, Oct 09, 2017 at 08:44:07PM +0300, Leonid Evdokimov wrote: > Hello! > > I'd like to see few more lines about SHA-2 support in glibc crypt() in > documentation. I've created small patch for nginx.org repo, it's > attached. > > -- > WBRBW, Leonid Evdokimov, xmpp:leon at darkk.net.ru http://darkk.net.ru tel:+79816800702 > PGP: 6691 DE6B 4CCD C1C1 76A0 0D4A E1F2 A980 7F50 FAB2 > # HG changeset patch > # User Leonid Evdokimov > # Date 1507567640 -10800 > # Mon Oct 09 19:47:20 2017 +0300 > # Node ID ef79dd801d1b7aedebd76c9e3a034b2009a34b78 > # Parent 2ac75347ed1af5a42d84e08edbe66f762676e668 > Mention that crypt() may support SHA-2 schemes as well > > diff -r 2ac75347ed1a -r ef79dd801d1b xml/en/docs/http/ngx_http_auth_basic_module.xml > --- a/xml/en/docs/http/ngx_http_auth_basic_module.xml Mon Oct 09 18:14:23 2017 +0300 > +++ b/xml/en/docs/http/ngx_http_auth_basic_module.xml Mon Oct 09 19:47:20 2017 +0300 > @@ -97,6 +97,12 @@ > encrypted with the crypt function; can be generated using > the ?htpasswd? utility from the Apache HTTP Server > distribution or the ?openssl passwd? command; > + > +Glibc implementation of crypt > +also support salted SHA-256 and SHA-512 schemes ($5$ ? $6$). > +These hashes may be generated using ?mkpasswd? utlity from ?whois? package or following python one-liner: > +?python2 -c 'import base64, os, crypt; print crypt.crypt("P at ssw0rd", "$6$" + base64.b64encode(os.urandom(12), altchars="./"))'?. All crypt() schemes available on a particular OS are supported, and this is what is written in the above paragraph. It is not clear why to document $5$ and $6$ explicitly. (Also, it might not be a good idea to actually use $5$ and especially $6$ crypt schemes for web authentication, as crypt() is needed for each request, and these schemes are quite CPU intensive.) [...] -- Maxim Dounin http://nginx.org/ From leon at darkk.net.ru Mon Oct 9 20:18:47 2017 From: leon at darkk.net.ru (Leonid Evdokimov) Date: Mon, 9 Oct 2017 23:18:47 +0300 Subject: [patch]: document SHA-2 support in glibc crypt() In-Reply-To: <20171009194411.GH75166@mdounin.ru> References: <20171009165402.GB5231@darkk-ya-laptop> <20171009165816.GA5388@darkk-ya-laptop> <20171009174407.GB5549@darkk-ya-laptop> <20171009194411.GH75166@mdounin.ru> Message-ID: <20171009201847.GA8279@darkk-ya-laptop> On Mon, Oct 09, 2017 at 10:44:11PM +0300, Maxim Dounin wrote: > All crypt() schemes available on a particular OS are supported, and > this is what is written in the above paragraph. I added that note to provide disambiguation that actual libc crypt() is used, I was under assumption that some only "plain old crypt()" is actually supported (like DES one) as the example does not refer to system crypt(), but refers to openssl and htpasswd. I was unaware of platform crypt() call till I have actually looked at the source code :) > It is not clear why to document $5$ and $6$ explicitly. That's just an example. These two are documented in crypt(3) manpage: MD5-based $1$ is already documented and $2a$ is not available in "default" build of glibc. > (Also, it might not be a good idea to actually use $5$ and especially > $6$ crypt schemes for web authentication, as crypt() is needed for > each request, and these schemes are quite CPU intensive.) Yep, that's true, that's 5000 rounds of SHA-2 and that's ~2..3ms of CPU time per request. -- WBRBW, Leonid Evdokimov, xmpp:leon at darkk.net.ru http://darkk.net.ru tel:+79816800702 PGP: 6691 DE6B 4CCD C1C1 76A0 0D4A E1F2 A980 7F50 FAB2 -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 819 bytes Desc: not available URL: From pluknet at nginx.com Mon Oct 9 22:27:57 2017 From: pluknet at nginx.com (Sergey Kandaurov) Date: Mon, 09 Oct 2017 22:27:57 +0000 Subject: [nginx] Upstream: fixed error handling of stale and revalidated cache send. Message-ID: details: http://hg.nginx.org/nginx/rev/77fe752b4586 branches: changeset: 7126:77fe752b4586 user: Sergey Kandaurov date: Tue Oct 10 01:04:54 2017 +0300 description: Upstream: fixed error handling of stale and revalidated cache send. The NGX_DONE value returned from ngx_http_upstream_cache_send() indicates that upstream was already finalized in ngx_http_upstream_process_headers(). It was treated as a generic error which resulted in duplicate finalization. Handled NGX_HTTP_UPSTREAM_INVALID_HEADER from ngx_http_upstream_cache_send(). Previously, it could return within ngx_http_upstream_finalize_request(), and since it's below NGX_HTTP_SPECIAL_RESPONSE, a client connection could stuck. diffstat: src/http/ngx_http_upstream.c | 42 ++++++++++++++++++++++++++++++++++++------ 1 files changed, 36 insertions(+), 6 deletions(-) diffs (66 lines): diff -r b8fc701daf8e -r 77fe752b4586 src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c Mon Oct 09 15:59:10 2017 +0300 +++ b/src/http/ngx_http_upstream.c Tue Oct 10 01:04:54 2017 +0300 @@ -2404,9 +2404,20 @@ ngx_http_upstream_test_next(ngx_http_req rc = u->reinit_request(r); - if (rc == NGX_OK) { - u->cache_status = NGX_HTTP_CACHE_STALE; - rc = ngx_http_upstream_cache_send(r, u); + if (rc != NGX_OK) { + ngx_http_upstream_finalize_request(r, u, rc); + return NGX_OK; + } + + u->cache_status = NGX_HTTP_CACHE_STALE; + rc = ngx_http_upstream_cache_send(r, u); + + if (rc == NGX_DONE) { + return NGX_OK; + } + + if (rc == NGX_HTTP_UPSTREAM_INVALID_HEADER) { + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; } ngx_http_upstream_finalize_request(r, u, rc); @@ -2444,6 +2455,14 @@ ngx_http_upstream_test_next(ngx_http_req u->cache_status = NGX_HTTP_CACHE_REVALIDATED; rc = ngx_http_upstream_cache_send(r, u); + if (rc == NGX_DONE) { + return NGX_OK; + } + + if (rc == NGX_HTTP_UPSTREAM_INVALID_HEADER) { + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + } + if (valid == 0) { valid = r->cache->valid_sec; updating = r->cache->updating_sec; @@ -4150,9 +4169,20 @@ ngx_http_upstream_next(ngx_http_request_ rc = u->reinit_request(r); - if (rc == NGX_OK) { - u->cache_status = NGX_HTTP_CACHE_STALE; - rc = ngx_http_upstream_cache_send(r, u); + if (rc != NGX_OK) { + ngx_http_upstream_finalize_request(r, u, rc); + return; + } + + u->cache_status = NGX_HTTP_CACHE_STALE; + rc = ngx_http_upstream_cache_send(r, u); + + if (rc == NGX_DONE) { + return; + } + + if (rc == NGX_HTTP_UPSTREAM_INVALID_HEADER) { + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; } ngx_http_upstream_finalize_request(r, u, rc); From mdounin at mdounin.ru Tue Oct 10 15:26:08 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 10 Oct 2017 15:26:08 +0000 Subject: [nginx] nginx-1.13.6-RELEASE Message-ID: details: http://hg.nginx.org/nginx/rev/f87da7d9ca02 branches: changeset: 7127:f87da7d9ca02 user: Maxim Dounin date: Tue Oct 10 18:22:50 2017 +0300 description: nginx-1.13.6-RELEASE diffstat: docs/xml/nginx/changes.xml | 120 +++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 120 insertions(+), 0 deletions(-) diffs (130 lines): diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,126 @@ + + + + +??? ????????????? ????????? ssl_preread +? ?????? stream ?? ???????? ???????????? ?? ????????? ??????. + + +switching to the next upstream server in the stream module did not work +when using the "ssl_preread" directive. + + + + + +? ?????? ngx_http_v2_module.
+??????? Piotr Sikora. + + +in the ngx_http_v2_module.
+Thanks to Piotr Sikora. + + + + + +nginx ?? ??????????? ???? ????? 2038 ???? +?? 32-?????? ?????????? ? 64-?????? time_t. + + +nginx did not support dates after the year 2038 +on 32-bit platforms with 64-bit time_t. + + + + + +? ????????? ??? ?? 1970 ???? ? ????? 10000 ????. + + +in handling of dates prior to the year 1970 and after the year 10000. + + + + + +? ?????? stream ???????? ???????? UDP-??????? ?? ???????? +?? ????????????? ??? ????????????? ?? ?????? info ?????? error. + + +in the stream module timeouts waiting for UDP datagrams from upstream servers +were not logged or logged at the "info" level instead of "error". + + + + + +??? ????????????? HTTP/2 nginx ??? ??????? ?????? 400, +?? ?????? ? ???? ???????. + + +when using HTTP/2 nginx might return the 400 response +without logging the reason. + + + + + +? ????????? ???????????? ?????? ????. + + +in processing of corrupted cache files. + + + + + +??? ??????????? ??????, ????????????? error_page, +?? ??????????? ????????? ?????????? ????????????. + + +cache control headers were ignored +when caching errors intercepted by error_page. + + + + + +??? ????????????? HTTP/2 ???? ??????? ????? ???? ??????????. + + +when using HTTP/2 client request body might be corrupted. + + + + + +? ????????? ??????? ???????? ??? ????????????? unix domain ???????. + + +in handling of client addresses when using unix domain sockets. + + + + + +??? ????????????? ????????? "hash ... consistent" ? ????? upstream +nginx ???????? ?????????, ???? ?????????????? ??????? ???? +? ??? ??? ????? ??? ??????? ???? ??????????. + + +nginx hogged CPU +when using the "hash ... consistent" directive in the upstream block +if large weights were used and all or most of the servers were unavailable. + + + + + + From mdounin at mdounin.ru Tue Oct 10 15:26:10 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 10 Oct 2017 15:26:10 +0000 Subject: [nginx] release-1.13.6 tag Message-ID: details: http://hg.nginx.org/nginx/rev/3012fcb69db4 branches: changeset: 7128:3012fcb69db4 user: Maxim Dounin date: Tue Oct 10 18:22:51 2017 +0300 description: release-1.13.6 tag diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -418,3 +418,4 @@ 5be2b25bdc65775a85f18f68a4be4f58c7384415 8457ce87640f9bfe6221c4ac4466ced20e03bebe release-1.13.3 bbc642c813c829963ce8197c0ca237ab7601f3d4 release-1.13.4 0d45b4cf7c2e4e626a5a16e1fe604402ace1cea5 release-1.13.5 +f87da7d9ca02b8ced4caa6c5eb9013ccd47b0117 release-1.13.6 From mdounin at mdounin.ru Tue Oct 10 17:29:43 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 10 Oct 2017 20:29:43 +0300 Subject: [patch]: document SHA-2 support in glibc crypt() In-Reply-To: <20171009201847.GA8279@darkk-ya-laptop> References: <20171009165402.GB5231@darkk-ya-laptop> <20171009165816.GA5388@darkk-ya-laptop> <20171009174407.GB5549@darkk-ya-laptop> <20171009194411.GH75166@mdounin.ru> <20171009201847.GA8279@darkk-ya-laptop> Message-ID: <20171010172943.GR75166@mdounin.ru> Hello! On Mon, Oct 09, 2017 at 11:18:47PM +0300, Leonid Evdokimov wrote: > On Mon, Oct 09, 2017 at 10:44:11PM +0300, Maxim Dounin wrote: > > All crypt() schemes available on a particular OS are supported, and > > this is what is written in the above paragraph. > > I added that note to provide disambiguation that actual libc crypt() is > used, I was under assumption that some only "plain old crypt()" is > actually supported (like DES one) as the example does not refer to > system crypt(), but refers to openssl and htpasswd. I was unaware of > platform crypt() call till I have actually looked at the source code :) The paragraph in question is expected to say that nginx uses the crypt() function as provided by system libraries. If it is not clear, we can consider improving the wording, and/or providing examples on how to use the tools mentioned to generate various types of passwords understood by crypt(). In particular, openssl by default generates traditional crypt() hashes, and can be used to generate $1$ hashes with the "-1" switch: $ openssl passwd foo GLJoKLSDZtEYU $ openssl passwd -1 foo $1$k8V9xFsq$y6xcPzRK5YW1QubxEm9kL1 (Not-yet-released openssl 1.1.1 also supports "-5" and "-6", though I would rather refrain from providing relevant examples.) > > It is not clear why to document $5$ and $6$ explicitly. > > That's just an example. These two are documented in crypt(3) manpage: > MD5-based $1$ is already documented and $2a$ is not available in > "default" build of glibc. It is not clear what you mean by saying "MD5-based $1$ is already documented". In nginx documentation there is nothing about $1$. There is a paragraph about $apr1$, Apache variant of $1$, which is similar, but is not crypt()-based - instead, it is explicitly implemented as a platform-independent solution which is available on all platforms including Windows. And this is why it is documented explicitly. -- Maxim Dounin http://nginx.org/ From czhihua at vmware.com Wed Oct 11 08:45:04 2017 From: czhihua at vmware.com (Zhihua Cao) Date: Wed, 11 Oct 2017 01:45:04 -0700 Subject: [PATCH] Free the shared memory only when reconfiguration is successful Message-ID: <648b1cca8f50d83eea02.1507711504@ubuntu> # HG changeset patch # User Zhihua Cao # Date 1507710209 25200 # Wed Oct 11 01:23:29 2017 -0700 # Node ID 648b1cca8f50d83eea02a6cc2c105ae95a3f3d72 # Parent 3012fcb69db4f35dd5851e3156625dc18a823fce Free the shared memory only when reconfiguration is successful If nginx reconfiguration fails, it maybe crash when killing the master process. The reason is that the unreused shared memory is freed during reconfiguration, the shared memory address is unmapped from master's address space, but the reconfiguration maybe fail(open listening sockets maybe fails due to the address is already inuse) and roll back to the old configuration. When killing nginx process, master still access the adrress which is unmapped from master's address space, segment fault exception occurs. diff -r 3012fcb69db4 -r 648b1cca8f50 src/core/ngx_cycle.c --- a/src/core/ngx_cycle.c Tue Oct 10 18:22:51 2017 +0300 +++ b/src/core/ngx_cycle.c Wed Oct 11 01:23:29 2017 -0700 @@ -16,6 +16,7 @@ static ngx_int_t ngx_test_lockfile(u_char *file, ngx_log_t *log); static void ngx_clean_old_cycles(ngx_event_t *ev); static void ngx_shutdown_timer_handler(ngx_event_t *ev); +static void ngx_shared_memory_reset_stale(ngx_cycle_t *cycle); volatile ngx_cycle_t *ngx_cycle; @@ -470,7 +471,8 @@ goto shm_zone_found; } - ngx_shm_free(&oshm_zone[n].shm); + /* don't free the old shm zone here, just mark it stale */ + oshm_zone[n].stale = 1; break; } @@ -648,6 +650,12 @@ i = 0; } + /* reconfiguration is successful, free the shared memory zone */ + if (oshm_zone[i].stale == 1) { + ngx_shm_free(&oshm_zone[i].shm); + continue; + } + part = &cycle->shared_memory.part; shm_zone = part->elts; @@ -798,6 +806,8 @@ failed: + ngx_shared_memory_reset_stale(old_cycle); + if (!ngx_is_init_cycle(old_cycle)) { old_ccf = (ngx_core_conf_t *) ngx_get_conf(old_cycle->conf_ctx, ngx_core_module); @@ -1201,6 +1211,34 @@ } +void +ngx_shared_memory_reset_stale(ngx_cycle_t *cycle) +{ + ngx_uint_t i; + ngx_list_part_t *part; + ngx_shm_zone_t *shm_zone; + + part = &cycle->shared_memory.part; + shm_zone = part->elts; + + for (i = 0; /* void */ ; i++) { + + if (i >= part->nelts) { + if (part->next == NULL) { + return; + } + part = part->next; + shm_zone = part->elts; + i = 0; + } + + if (shm_zone[i].stale == 1) { + shm_zone[i].stale = 0; + } + } +} + + ngx_shm_zone_t * ngx_shared_memory_add(ngx_conf_t *cf, ngx_str_t *name, size_t size, void *tag) { @@ -1269,6 +1307,7 @@ shm_zone->init = NULL; shm_zone->tag = tag; shm_zone->noreuse = 0; + shm_zone->stale = 0; return shm_zone; } diff -r 3012fcb69db4 -r 648b1cca8f50 src/core/ngx_cycle.h --- a/src/core/ngx_cycle.h Tue Oct 10 18:22:51 2017 +0300 +++ b/src/core/ngx_cycle.h Wed Oct 11 01:23:29 2017 -0700 @@ -31,7 +31,8 @@ ngx_shm_t shm; ngx_shm_zone_init_pt init; void *tag; - ngx_uint_t noreuse; /* unsigned noreuse:1; */ + ngx_uint_t noreuse:1; /* unsigned noreuse:1; */ + ngx_uint_t stale:1; }; From czhihua at vmware.com Wed Oct 11 08:55:17 2017 From: czhihua at vmware.com (Cao Zhihua) Date: Wed, 11 Oct 2017 08:55:17 +0000 Subject: Free the shared memory only when reconfiguration is successful Message-ID: # HG changeset patch # User Zhihua Cao # Date 1507710209 25200 # Wed Oct 11 01:23:29 2017 -0700 # Node ID 648b1cca8f50d83eea02a6cc2c105ae95a3f3d72 # Parent 3012fcb69db4f35dd5851e3156625dc18a823fce Free the shared memory only when reconfiguration is successful If nginx reconfiguration fails, it maybe crash when killing the master process. The reason is that the unreused shared memory is freed during reconfiguration, the shared memory address is unmapped from master's address space, but the reconfiguration maybe fail(open listening sockets maybe fails due to the address is already inuse) and roll back to the old configuration. When killing nginx process, master still access the adrress which is unmapped from master's address space, segment fault exception occurs. diff -r 3012fcb69db4 -r 648b1cca8f50 src/core/ngx_cycle.c --- a/src/core/ngx_cycle.c Tue Oct 10 18:22:51 2017 +0300 +++ b/src/core/ngx_cycle.c Wed Oct 11 01:23:29 2017 -0700 @@ -16,6 +16,7 @@ static ngx_int_t ngx_test_lockfile(u_char *file, ngx_log_t *log); static void ngx_clean_old_cycles(ngx_event_t *ev); static void ngx_shutdown_timer_handler(ngx_event_t *ev); +static void ngx_shared_memory_reset_stale(ngx_cycle_t *cycle); volatile ngx_cycle_t *ngx_cycle; @@ -470,7 +471,8 @@ goto shm_zone_found; } - ngx_shm_free(&oshm_zone[n].shm); + /* don't free the old shm zone here, just mark it stale */ + oshm_zone[n].stale = 1; break; } @@ -648,6 +650,12 @@ i = 0; } + /* reconfiguration is successful, free the shared memory zone */ + if (oshm_zone[i].stale == 1) { + ngx_shm_free(&oshm_zone[i].shm); + continue; + } + part = &cycle->shared_memory.part; shm_zone = part->elts; @@ -798,6 +806,8 @@ failed: + ngx_shared_memory_reset_stale(old_cycle); + if (!ngx_is_init_cycle(old_cycle)) { old_ccf = (ngx_core_conf_t *) ngx_get_conf(old_cycle->conf_ctx, ngx_core_module); @@ -1201,6 +1211,34 @@ } +void +ngx_shared_memory_reset_stale(ngx_cycle_t *cycle) +{ + ngx_uint_t i; + ngx_list_part_t *part; + ngx_shm_zone_t *shm_zone; + + part = &cycle->shared_memory.part; + shm_zone = part->elts; + + for (i = 0; /* void */ ; i++) { + + if (i >= part->nelts) { + if (part->next == NULL) { + return; + } + part = part->next; + shm_zone = part->elts; + i = 0; + } + + if (shm_zone[i].stale == 1) { + shm_zone[i].stale = 0; + } + } +} + + ngx_shm_zone_t * ngx_shared_memory_add(ngx_conf_t *cf, ngx_str_t *name, size_t size, void *tag) { @@ -1269,6 +1307,7 @@ shm_zone->init = NULL; shm_zone->tag = tag; shm_zone->noreuse = 0; + shm_zone->stale = 0; return shm_zone; } diff -r 3012fcb69db4 -r 648b1cca8f50 src/core/ngx_cycle.h --- a/src/core/ngx_cycle.h Tue Oct 10 18:22:51 2017 +0300 +++ b/src/core/ngx_cycle.h Wed Oct 11 01:23:29 2017 -0700 @@ -31,7 +31,8 @@ ngx_shm_t shm; ngx_shm_zone_init_pt init; void *tag; - ngx_uint_t noreuse; /* unsigned noreuse:1; */ + ngx_uint_t noreuse:1; /* unsigned noreuse:1; */ + ngx_uint_t stale:1; }; -------------- next part -------------- An HTML attachment was scrubbed... URL: From xeioex at nginx.com Wed Oct 11 10:16:11 2017 From: xeioex at nginx.com (Dmitry Volyntsev) Date: Wed, 11 Oct 2017 13:16:11 +0300 Subject: [njs] Version 0.1.14. In-Reply-To: <20171011030743.GD99496@FreeBSD.org.ru> References: <20171011030743.GD99496@FreeBSD.org.ru> Message-ID: On 11.10.2017 06:07, Sergey A. Osokin wrote: > Hi Dmitry, > > recent commits in njs module including this one are unavailable in > github.com/nginx/njs. Could you please take a look. > > Thanks in advance. > Hi Sergey, github export is fixed now and the recent commits are available. Thank you for reporting the issue. From mdounin at mdounin.ru Wed Oct 11 17:38:45 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Wed, 11 Oct 2017 20:38:45 +0300 Subject: [PATCH] Free the shared memory only when reconfiguration is successful In-Reply-To: <648b1cca8f50d83eea02.1507711504@ubuntu> References: <648b1cca8f50d83eea02.1507711504@ubuntu> Message-ID: <20171011173844.GX75166@mdounin.ru> Hello! On Wed, Oct 11, 2017 at 01:45:04AM -0700, Zhihua Cao wrote: > # HG changeset patch > # User Zhihua Cao > # Date 1507710209 25200 > # Wed Oct 11 01:23:29 2017 -0700 > # Node ID 648b1cca8f50d83eea02a6cc2c105ae95a3f3d72 > # Parent 3012fcb69db4f35dd5851e3156625dc18a823fce > Free the shared memory only when reconfiguration is successful > > If nginx reconfiguration fails, it maybe crash when killing the master process. > The reason is that the unreused shared memory is freed during reconfiguration, > the shared memory address is unmapped from master's address space, but the > reconfiguration maybe fail(open listening sockets maybe fails due to the > address is already inuse) and roll back to the old configuration. > When killing nginx process, master still access the adrress > which is unmapped from master's address space, segment fault exception occurs. Thank you for the patch. See comments below. > diff -r 3012fcb69db4 -r 648b1cca8f50 src/core/ngx_cycle.c > --- a/src/core/ngx_cycle.c Tue Oct 10 18:22:51 2017 +0300 > +++ b/src/core/ngx_cycle.c Wed Oct 11 01:23:29 2017 -0700 > @@ -16,6 +16,7 @@ > static ngx_int_t ngx_test_lockfile(u_char *file, ngx_log_t *log); > static void ngx_clean_old_cycles(ngx_event_t *ev); > static void ngx_shutdown_timer_handler(ngx_event_t *ev); > +static void ngx_shared_memory_reset_stale(ngx_cycle_t *cycle); > > > volatile ngx_cycle_t *ngx_cycle; > @@ -470,7 +471,8 @@ > goto shm_zone_found; > } > > - ngx_shm_free(&oshm_zone[n].shm); > + /* don't free the old shm zone here, just mark it stale */ > + oshm_zone[n].stale = 1; > > break; > } There should be no need to introduce additional the "stale" flag and set/reset it. Simply using identical conditions in the "free the unnecessary shared memory" loop should work fine. See patch below (mostly untested though). Another problem to consider here are platforms where shared memory zone name is actually used externally, notably Windows. On the other hand, re-creating shared memory zone with the same name seems to be already silently broken at least on Windows: it simply uses the old zone instead, potentially leading to a segmentation fault as well. With the patch, it now properly fails. # HG changeset patch # User Maxim Dounin # Date 1507739571 -10800 # Wed Oct 11 19:32:51 2017 +0300 # Node ID 7e1cdcd9e88283c0ba6883a4bbc73fd784a4cf19 # Parent 3012fcb69db4f35dd5851e3156625dc18a823fce Core: free shared memory zones only after reconfiguration. This is what usually happens for zones no longer used in the new configuration, but zones where size or tag were changed were freed when creating a new memory zones. If reconfiguration failed (for example, due a conflicting listening socket), this resulted in a segmentation fault in the master process. Reported by Zhihua Cao, http://mailman.nginx.org/pipermail/nginx-devel/2017-October/010536.html. diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c --- a/src/core/ngx_cycle.c +++ b/src/core/ngx_cycle.c @@ -470,8 +470,6 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) goto shm_zone_found; } - ngx_shm_free(&oshm_zone[n].shm); - break; } @@ -662,14 +660,26 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) n = 0; } - if (oshm_zone[i].shm.name.len == shm_zone[n].shm.name.len - && ngx_strncmp(oshm_zone[i].shm.name.data, - shm_zone[n].shm.name.data, - oshm_zone[i].shm.name.len) - == 0) + if (oshm_zone[i].shm.name.len != shm_zone[n].shm.name.len) { + continue; + } + + if (ngx_strncmp(oshm_zone[i].shm.name.data, + shm_zone[n].shm.name.data, + oshm_zone[i].shm.name.len) + != 0) + { + continue; + } + + if (oshm_zone[i].tag == shm_zone[n].tag + && oshm_zone[i].shm.size == shm_zone[n].shm.size + && !oshm_zone[i].noreuse) { goto live_shm_zone; } + + break; } ngx_shm_free(&oshm_zone[i].shm); -- Maxim Dounin http://nginx.org/ From ru at nginx.com Wed Oct 11 19:18:30 2017 From: ru at nginx.com (Ruslan Ermilov) Date: Wed, 11 Oct 2017 19:18:30 +0000 Subject: [nginx] Version bump. Message-ID: details: http://hg.nginx.org/nginx/rev/f4cf4f970e6e branches: changeset: 7129:f4cf4f970e6e user: Ruslan Ermilov date: Wed Oct 11 22:04:11 2017 +0300 description: Version bump. diffstat: src/core/nginx.h | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 3012fcb69db4 -r f4cf4f970e6e src/core/nginx.h --- a/src/core/nginx.h Tue Oct 10 18:22:51 2017 +0300 +++ b/src/core/nginx.h Wed Oct 11 22:04:11 2017 +0300 @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1013006 -#define NGINX_VERSION "1.13.6" +#define nginx_version 1013007 +#define NGINX_VERSION "1.13.7" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From ru at nginx.com Wed Oct 11 19:18:32 2017 From: ru at nginx.com (Ruslan Ermilov) Date: Wed, 11 Oct 2017 19:18:32 +0000 Subject: [nginx] Upstream: fixed $upstream_status when upstream returns 503/504. Message-ID: details: http://hg.nginx.org/nginx/rev/882ad033d43c branches: changeset: 7130:882ad033d43c user: Ruslan Ermilov date: Wed Oct 11 22:04:28 2017 +0300 description: Upstream: fixed $upstream_status when upstream returns 503/504. If proxy_next_upstream includes http_503/http_504, and upstream returns 503/504, $upstream_status converted this to 502 for any values except the last one. diffstat: src/http/ngx_http_upstream.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (22 lines): diff -r f4cf4f970e6e -r 882ad033d43c src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c Wed Oct 11 22:04:11 2017 +0300 +++ b/src/http/ngx_http_upstream.c Wed Oct 11 22:04:28 2017 +0300 @@ -4111,6 +4111,7 @@ ngx_http_upstream_next(ngx_http_request_ switch (ft_type) { case NGX_HTTP_UPSTREAM_FT_TIMEOUT: + case NGX_HTTP_UPSTREAM_FT_HTTP_504: status = NGX_HTTP_GATEWAY_TIME_OUT; break; @@ -4118,6 +4119,10 @@ ngx_http_upstream_next(ngx_http_request_ status = NGX_HTTP_INTERNAL_SERVER_ERROR; break; + case NGX_HTTP_UPSTREAM_FT_HTTP_503: + status = NGX_HTTP_SERVICE_UNAVAILABLE; + break; + case NGX_HTTP_UPSTREAM_FT_HTTP_403: status = NGX_HTTP_FORBIDDEN; break; From alessandro at ghedini.me Wed Oct 11 22:58:37 2017 From: alessandro at ghedini.me (Alessandro Ghedini) Date: Wed, 11 Oct 2017 15:58:37 -0700 Subject: [PATCH] SSL: include Message-ID: # HG changeset patch # User Alessandro Ghedini # Date 1507761830 25200 # Wed Oct 11 15:43:50 2017 -0700 # Node ID fe296bd63e074759392076a2bb6b36f77301227d # Parent fe7d9e3987d40f16d86fd01d94ad16ff58467af2 SSL: include This header carries the definition of HMAC_Init_ex(). In OpenSSL this header is included by , but it's not so in BoringSSL. It's probably a good idea to explicitly include this header anyway, regardless of whether it's included by other headers or not. diff -r fe7d9e3987d4 -r fe296bd63e07 src/event/ngx_event_openssl.h --- a/src/event/ngx_event_openssl.h Wed Sep 28 14:54:44 2016 +0100 +++ b/src/event/ngx_event_openssl.h Wed Oct 11 15:43:50 2017 -0700 @@ -22,6 +22,7 @@ #include #endif #include +#include #ifndef OPENSSL_NO_OCSP #include #endif From arut at nginx.com Thu Oct 12 08:58:09 2017 From: arut at nginx.com (Roman Arutyunyan) Date: Thu, 12 Oct 2017 08:58:09 +0000 Subject: [nginx] Upstream: disabled upgrading in subrequests. Message-ID: details: http://hg.nginx.org/nginx/rev/6a5a91de5b74 branches: changeset: 7131:6a5a91de5b74 user: Roman Arutyunyan date: Wed Oct 11 17:38:21 2017 +0300 description: Upstream: disabled upgrading in subrequests. Upgrading an upstream connection is usually followed by reading from the client which a subrequest is not allowed to do. Moreover, accessing the header_in request field while processing upgraded connection ends up with a null pointer dereference since the header_in buffer is only created for the the main request. diffstat: src/http/ngx_http_upstream.c | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diffs (17 lines): diff -r 882ad033d43c -r 6a5a91de5b74 src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c Wed Oct 11 22:04:28 2017 +0300 +++ b/src/http/ngx_http_upstream.c Wed Oct 11 17:38:21 2017 +0300 @@ -3206,6 +3206,13 @@ ngx_http_upstream_upgrade(ngx_http_reque /* TODO: prevent upgrade if not requested or not possible */ + if (r != r->main) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "connection upgrade in subrequest"); + ngx_http_upstream_finalize_request(r, u, NGX_ERROR); + return; + } + r->keepalive = 0; c->log->action = "proxying upgraded connection"; From mdounin at mdounin.ru Thu Oct 12 15:12:36 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 12 Oct 2017 15:12:36 +0000 Subject: [nginx] SSL: include . Message-ID: details: http://hg.nginx.org/nginx/rev/8076ba459f05 branches: changeset: 7132:8076ba459f05 user: Alessandro Ghedini date: Wed Oct 11 15:43:50 2017 -0700 description: SSL: include . This header carries the definition of HMAC_Init_ex(). In OpenSSL this header is included by , but it's not so in BoringSSL. It's probably a good idea to explicitly include this header anyway, regardless of whether it's included by other headers or not. diffstat: src/event/ngx_event_openssl.h | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -22,6 +22,7 @@ #include #endif #include +#include #ifndef OPENSSL_NO_OCSP #include #endif From mdounin at mdounin.ru Thu Oct 12 15:13:29 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Thu, 12 Oct 2017 18:13:29 +0300 Subject: [PATCH] SSL: include In-Reply-To: References: Message-ID: <20171012151329.GG75166@mdounin.ru> Hello! On Wed, Oct 11, 2017 at 03:58:37PM -0700, Alessandro Ghedini wrote: > # HG changeset patch > # User Alessandro Ghedini > # Date 1507761830 25200 > # Wed Oct 11 15:43:50 2017 -0700 > # Node ID fe296bd63e074759392076a2bb6b36f77301227d > # Parent fe7d9e3987d40f16d86fd01d94ad16ff58467af2 > SSL: include > > This header carries the definition of HMAC_Init_ex(). In OpenSSL this > header is included by , but it's not so in BoringSSL. > > It's probably a good idea to explicitly include this header anyway, > regardless of whether it's included by other headers or not. > > diff -r fe7d9e3987d4 -r fe296bd63e07 src/event/ngx_event_openssl.h > --- a/src/event/ngx_event_openssl.h Wed Sep 28 14:54:44 2016 +0100 > +++ b/src/event/ngx_event_openssl.h Wed Oct 11 15:43:50 2017 -0700 > @@ -22,6 +22,7 @@ > #include > #endif > #include > +#include > #ifndef OPENSSL_NO_OCSP > #include > #endif Committed (with a dot added to the summary line), thanks. http://hg.nginx.org/nginx/rev/8076ba459f05 -- Maxim Dounin http://nginx.org/ From ru at nginx.com Thu Oct 12 21:32:45 2017 From: ru at nginx.com (Ruslan Ermilov) Date: Thu, 12 Oct 2017 21:32:45 +0000 Subject: [nginx] Fixed type of ngx_conf_t.handler_conf. Message-ID: details: http://hg.nginx.org/nginx/rev/dc3b3cfd5d23 branches: changeset: 7133:dc3b3cfd5d23 user: Ruslan Ermilov date: Fri Oct 13 00:32:26 2017 +0300 description: Fixed type of ngx_conf_t.handler_conf. The type should have been changed in c9b243802a17 along with changing ngx_conf_handler_pt. diffstat: src/core/ngx_conf_file.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 8076ba459f05 -r dc3b3cfd5d23 src/core/ngx_conf_file.h --- a/src/core/ngx_conf_file.h Wed Oct 11 15:43:50 2017 -0700 +++ b/src/core/ngx_conf_file.h Fri Oct 13 00:32:26 2017 +0300 @@ -128,7 +128,7 @@ struct ngx_conf_s { ngx_uint_t cmd_type; ngx_conf_handler_pt handler; - char *handler_conf; + void *handler_conf; }; From czhihua at vmware.com Fri Oct 13 08:43:16 2017 From: czhihua at vmware.com (Cao Zhihua) Date: Fri, 13 Oct 2017 08:43:16 +0000 Subject: nginx-devel Digest, Vol 96, Issue 10 In-Reply-To: References: Message-ID: Hi Maxim, Your patch is fine to me. Thanks Zhihua Message: 1 Date: Wed, 11 Oct 2017 20:38:45 +0300 From: Maxim Dounin To: nginx-devel at nginx.org Subject: Re: [PATCH] Free the shared memory only when reconfiguration is successful Message-ID: <20171011173844.GX75166 at mdounin.ru> Content-Type: text/plain; charset=us-ascii Hello! On Wed, Oct 11, 2017 at 01:45:04AM -0700, Zhihua Cao wrote: > # HG changeset patch > # User Zhihua Cao # Date 1507710209 25200 > # Wed Oct 11 01:23:29 2017 -0700 > # Node ID 648b1cca8f50d83eea02a6cc2c105ae95a3f3d72 > # Parent 3012fcb69db4f35dd5851e3156625dc18a823fce > Free the shared memory only when reconfiguration is successful > > If nginx reconfiguration fails, it maybe crash when killing the master process. > The reason is that the unreused shared memory is freed during > reconfiguration, the shared memory address is unmapped from master's > address space, but the reconfiguration maybe fail(open listening > sockets maybe fails due to the address is already inuse) and roll back to the old configuration. > When killing nginx process, master still access the adrress which is > unmapped from master's address space, segment fault exception occurs. Thank you for the patch. See comments below. > diff -r 3012fcb69db4 -r 648b1cca8f50 src/core/ngx_cycle.c > --- a/src/core/ngx_cycle.c Tue Oct 10 18:22:51 2017 +0300 > +++ b/src/core/ngx_cycle.c Wed Oct 11 01:23:29 2017 -0700 > @@ -16,6 +16,7 @@ > static ngx_int_t ngx_test_lockfile(u_char *file, ngx_log_t *log); > static void ngx_clean_old_cycles(ngx_event_t *ev); static void > ngx_shutdown_timer_handler(ngx_event_t *ev); > +static void ngx_shared_memory_reset_stale(ngx_cycle_t *cycle); > > > volatile ngx_cycle_t *ngx_cycle; > @@ -470,7 +471,8 @@ > goto shm_zone_found; > } > > - ngx_shm_free(&oshm_zone[n].shm); > + /* don't free the old shm zone here, just mark it stale */ > + oshm_zone[n].stale = 1; > > break; > } There should be no need to introduce additional the "stale" flag and set/reset it. Simply using identical conditions in the "free the unnecessary shared memory" loop should work fine. See patch below (mostly untested though). Another problem to consider here are platforms where shared memory zone name is actually used externally, notably Windows. On the other hand, re-creating shared memory zone with the same name seems to be already silently broken at least on Windows: it simply uses the old zone instead, potentially leading to a segmentation fault as well. With the patch, it now properly fails. # HG changeset patch # User Maxim Dounin # Date 1507739571 -10800 # Wed Oct 11 19:32:51 2017 +0300 # Node ID 7e1cdcd9e88283c0ba6883a4bbc73fd784a4cf19 # Parent 3012fcb69db4f35dd5851e3156625dc18a823fce Core: free shared memory zones only after reconfiguration. This is what usually happens for zones no longer used in the new configuration, but zones where size or tag were changed were freed when creating a new memory zones. If reconfiguration failed (for example, due a conflicting listening socket), this resulted in a segmentation fault in the master process. Reported by Zhihua Cao, https://urldefense.proofpoint.com/v2/url?u=http-3A__mailman.nginx.org_pipermail_nginx-2Ddevel_2017-2DOctober_010536.html&d=DwICAg&c=uilaK90D4TOVoH58JNXRgQ&r=5hIR0vaOPz44U23VdaJSuH4pwdxinKp20laDxq322wI&m=s1p5j--5wkDOKiGinpToFlM7fKF9_zlVlcGwZe5_RD8&s=CPgxx9B73_SnhhF7jlX6DhY2qtIAmLtUA-2lu4OLeCw&e=. diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c --- a/src/core/ngx_cycle.c +++ b/src/core/ngx_cycle.c @@ -470,8 +470,6 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) goto shm_zone_found; } - ngx_shm_free(&oshm_zone[n].shm); - break; } @@ -662,14 +660,26 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) n = 0; } - if (oshm_zone[i].shm.name.len == shm_zone[n].shm.name.len - && ngx_strncmp(oshm_zone[i].shm.name.data, - shm_zone[n].shm.name.data, - oshm_zone[i].shm.name.len) - == 0) + if (oshm_zone[i].shm.name.len != shm_zone[n].shm.name.len) { + continue; + } + + if (ngx_strncmp(oshm_zone[i].shm.name.data, + shm_zone[n].shm.name.data, + oshm_zone[i].shm.name.len) + != 0) + { + continue; + } + + if (oshm_zone[i].tag == shm_zone[n].tag + && oshm_zone[i].shm.size == shm_zone[n].shm.size + && !oshm_zone[i].noreuse) { goto live_shm_zone; } + + break; } ngx_shm_free(&oshm_zone[i].shm); -- Maxim Dounin https://urldefense.proofpoint.com/v2/url?u=http-3A__nginx.org_&d=DwICAg&c=uilaK90D4TOVoH58JNXRgQ&r=5hIR0vaOPz44U23VdaJSuH4pwdxinKp20laDxq322wI&m=s1p5j--5wkDOKiGinpToFlM7fKF9_zlVlcGwZe5_RD8&s=Lhq1TAuEHcsGBdPKHCvpqkS_CYbBVJKBm00BqKcSQVk&e= From czhihua at vmware.com Fri Oct 13 10:13:03 2017 From: czhihua at vmware.com (Zhihua Cao) Date: Fri, 13 Oct 2017 03:13:03 -0700 Subject: [PATCH] Free the shared memory only when reconfiguration is successful Message-ID: <648b1cca8f50d83eea02.1507889583@ubuntu> # HG changeset patch # User Zhihua Cao # Date 1507710209 25200 # Wed Oct 11 01:23:29 2017 -0700 # Node ID 648b1cca8f50d83eea02a6cc2c105ae95a3f3d72 # Parent 3012fcb69db4f35dd5851e3156625dc18a823fce Free the shared memory only when reconfiguration is successful If nginx reconfiguration fails, it maybe crash when killing the master process. The reason is that the unreused shared memory is freed during reconfiguration, the shared memory address is unmapped from master's address space, but the reconfiguration maybe fail(open listening sockets maybe fails due to the address is already inuse) and roll back to the old configuration. When killing nginx process, master still access the adrress which is unmapped from master's address space, segment fault exception occurs. diff -r 3012fcb69db4 -r 648b1cca8f50 src/core/ngx_cycle.c --- a/src/core/ngx_cycle.c Tue Oct 10 18:22:51 2017 +0300 +++ b/src/core/ngx_cycle.c Wed Oct 11 01:23:29 2017 -0700 @@ -16,6 +16,7 @@ static ngx_int_t ngx_test_lockfile(u_char *file, ngx_log_t *log); static void ngx_clean_old_cycles(ngx_event_t *ev); static void ngx_shutdown_timer_handler(ngx_event_t *ev); +static void ngx_shared_memory_reset_stale(ngx_cycle_t *cycle); volatile ngx_cycle_t *ngx_cycle; @@ -470,7 +471,8 @@ goto shm_zone_found; } - ngx_shm_free(&oshm_zone[n].shm); + /* don't free the old shm zone here, just mark it stale */ + oshm_zone[n].stale = 1; break; } @@ -648,6 +650,12 @@ i = 0; } + /* reconfiguration is successful, free the shared memory zone */ + if (oshm_zone[i].stale == 1) { + ngx_shm_free(&oshm_zone[i].shm); + continue; + } + part = &cycle->shared_memory.part; shm_zone = part->elts; @@ -798,6 +806,8 @@ failed: + ngx_shared_memory_reset_stale(old_cycle); + if (!ngx_is_init_cycle(old_cycle)) { old_ccf = (ngx_core_conf_t *) ngx_get_conf(old_cycle->conf_ctx, ngx_core_module); @@ -1201,6 +1211,34 @@ } +void +ngx_shared_memory_reset_stale(ngx_cycle_t *cycle) +{ + ngx_uint_t i; + ngx_list_part_t *part; + ngx_shm_zone_t *shm_zone; + + part = &cycle->shared_memory.part; + shm_zone = part->elts; + + for (i = 0; /* void */ ; i++) { + + if (i >= part->nelts) { + if (part->next == NULL) { + return; + } + part = part->next; + shm_zone = part->elts; + i = 0; + } + + if (shm_zone[i].stale == 1) { + shm_zone[i].stale = 0; + } + } +} + + ngx_shm_zone_t * ngx_shared_memory_add(ngx_conf_t *cf, ngx_str_t *name, size_t size, void *tag) { @@ -1269,6 +1307,7 @@ shm_zone->init = NULL; shm_zone->tag = tag; shm_zone->noreuse = 0; + shm_zone->stale = 0; return shm_zone; } diff -r 3012fcb69db4 -r 648b1cca8f50 src/core/ngx_cycle.h --- a/src/core/ngx_cycle.h Tue Oct 10 18:22:51 2017 +0300 +++ b/src/core/ngx_cycle.h Wed Oct 11 01:23:29 2017 -0700 @@ -31,7 +31,8 @@ ngx_shm_t shm; ngx_shm_zone_init_pt init; void *tag; - ngx_uint_t noreuse; /* unsigned noreuse:1; */ + ngx_uint_t noreuse:1; /* unsigned noreuse:1; */ + ngx_uint_t stale:1; }; From czhihua at vmware.com Fri Oct 13 10:13:24 2017 From: czhihua at vmware.com (Zhihua Cao) Date: Fri, 13 Oct 2017 03:13:24 -0700 Subject: [PATCH] Free the shared memory only when reconfiguration is successful Message-ID: <648b1cca8f50d83eea02.1507889604@ubuntu> # HG changeset patch # User Zhihua Cao # Date 1507710209 25200 # Wed Oct 11 01:23:29 2017 -0700 # Node ID 648b1cca8f50d83eea02a6cc2c105ae95a3f3d72 # Parent 3012fcb69db4f35dd5851e3156625dc18a823fce Free the shared memory only when reconfiguration is successful If nginx reconfiguration fails, it maybe crash when killing the master process. The reason is that the unreused shared memory is freed during reconfiguration, the shared memory address is unmapped from master's address space, but the reconfiguration maybe fail(open listening sockets maybe fails due to the address is already inuse) and roll back to the old configuration. When killing nginx process, master still access the adrress which is unmapped from master's address space, segment fault exception occurs. diff -r 3012fcb69db4 -r 648b1cca8f50 src/core/ngx_cycle.c --- a/src/core/ngx_cycle.c Tue Oct 10 18:22:51 2017 +0300 +++ b/src/core/ngx_cycle.c Wed Oct 11 01:23:29 2017 -0700 @@ -16,6 +16,7 @@ static ngx_int_t ngx_test_lockfile(u_char *file, ngx_log_t *log); static void ngx_clean_old_cycles(ngx_event_t *ev); static void ngx_shutdown_timer_handler(ngx_event_t *ev); +static void ngx_shared_memory_reset_stale(ngx_cycle_t *cycle); volatile ngx_cycle_t *ngx_cycle; @@ -470,7 +471,8 @@ goto shm_zone_found; } - ngx_shm_free(&oshm_zone[n].shm); + /* don't free the old shm zone here, just mark it stale */ + oshm_zone[n].stale = 1; break; } @@ -648,6 +650,12 @@ i = 0; } + /* reconfiguration is successful, free the shared memory zone */ + if (oshm_zone[i].stale == 1) { + ngx_shm_free(&oshm_zone[i].shm); + continue; + } + part = &cycle->shared_memory.part; shm_zone = part->elts; @@ -798,6 +806,8 @@ failed: + ngx_shared_memory_reset_stale(old_cycle); + if (!ngx_is_init_cycle(old_cycle)) { old_ccf = (ngx_core_conf_t *) ngx_get_conf(old_cycle->conf_ctx, ngx_core_module); @@ -1201,6 +1211,34 @@ } +void +ngx_shared_memory_reset_stale(ngx_cycle_t *cycle) +{ + ngx_uint_t i; + ngx_list_part_t *part; + ngx_shm_zone_t *shm_zone; + + part = &cycle->shared_memory.part; + shm_zone = part->elts; + + for (i = 0; /* void */ ; i++) { + + if (i >= part->nelts) { + if (part->next == NULL) { + return; + } + part = part->next; + shm_zone = part->elts; + i = 0; + } + + if (shm_zone[i].stale == 1) { + shm_zone[i].stale = 0; + } + } +} + + ngx_shm_zone_t * ngx_shared_memory_add(ngx_conf_t *cf, ngx_str_t *name, size_t size, void *tag) { @@ -1269,6 +1307,7 @@ shm_zone->init = NULL; shm_zone->tag = tag; shm_zone->noreuse = 0; + shm_zone->stale = 0; return shm_zone; } diff -r 3012fcb69db4 -r 648b1cca8f50 src/core/ngx_cycle.h --- a/src/core/ngx_cycle.h Tue Oct 10 18:22:51 2017 +0300 +++ b/src/core/ngx_cycle.h Wed Oct 11 01:23:29 2017 -0700 @@ -31,7 +31,8 @@ ngx_shm_t shm; ngx_shm_zone_init_pt init; void *tag; - ngx_uint_t noreuse; /* unsigned noreuse:1; */ + ngx_uint_t noreuse:1; /* unsigned noreuse:1; */ + ngx_uint_t stale:1; }; From czhihua at vmware.com Fri Oct 13 10:14:00 2017 From: czhihua at vmware.com (Zhihua Cao) Date: Fri, 13 Oct 2017 03:14:00 -0700 Subject: [PATCH] Make ssl upstream server name check configurable Message-ID: # HG changeset patch # User Zhihua Cao # Date 1507889088 25200 # Fri Oct 13 03:04:48 2017 -0700 # Node ID d3ea6fe7edb19f55896ec1b77f76d23b7fb598a4 # Parent 648b1cca8f50d83eea02a6cc2c105ae95a3f3d72 Make ssl upstream server name check configurable Now when nginx always check common name in upstream's certificate with upstream.ssl_name. But they are not always same, it check fails, ssl handshake will fail. If proxy_ssl_server_name_check is off, turn off the check. diff -r 648b1cca8f50 -r d3ea6fe7edb1 src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c Wed Oct 11 01:23:29 2017 -0700 +++ b/src/http/modules/ngx_http_proxy_module.c Fri Oct 13 03:04:48 2017 -0700 @@ -673,6 +673,13 @@ offsetof(ngx_http_proxy_loc_conf_t, upstream.ssl_verify), NULL }, + { ngx_string("proxy_ssl_server_name_check"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, upstream.ssl_server_name_check), + NULL }, + { ngx_string("proxy_ssl_verify_depth"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, @@ -2906,6 +2913,7 @@ conf->upstream.ssl_session_reuse = NGX_CONF_UNSET; conf->upstream.ssl_server_name = NGX_CONF_UNSET; conf->upstream.ssl_verify = NGX_CONF_UNSET; + conf->upstream.ssl_server_name_check = NGX_CONF_UNSET; conf->ssl_verify_depth = NGX_CONF_UNSET_UINT; conf->ssl_passwords = NGX_CONF_UNSET_PTR; #endif @@ -3237,6 +3245,8 @@ prev->upstream.ssl_server_name, 0); ngx_conf_merge_value(conf->upstream.ssl_verify, prev->upstream.ssl_verify, 0); + ngx_conf_merge_value(conf->upstream.ssl_server_name_check, + prev->upstream.ssl_server_name_check, 1); ngx_conf_merge_uint_value(conf->ssl_verify_depth, prev->ssl_verify_depth, 1); ngx_conf_merge_str_value(conf->ssl_trusted_certificate, diff -r 648b1cca8f50 -r d3ea6fe7edb1 src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c Wed Oct 11 01:23:29 2017 -0700 +++ b/src/http/ngx_http_upstream.c Fri Oct 13 03:04:48 2017 -0700 @@ -1733,7 +1733,8 @@ goto failed; } - if (ngx_ssl_check_host(c, &u->ssl_name) != NGX_OK) { + if (u->conf->ssl_server_name_check + && ngx_ssl_check_host(c, &u->ssl_name) != NGX_OK) { ngx_log_error(NGX_LOG_ERR, c->log, 0, "upstream SSL certificate does not match \"%V\"", &u->ssl_name); diff -r 648b1cca8f50 -r d3ea6fe7edb1 src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h Wed Oct 11 01:23:29 2017 -0700 +++ b/src/http/ngx_http_upstream.h Fri Oct 13 03:04:48 2017 -0700 @@ -229,6 +229,7 @@ ngx_http_complex_value_t *ssl_name; ngx_flag_t ssl_server_name; ngx_flag_t ssl_verify; + ngx_flag_t ssl_server_name_check; #endif ngx_str_t module; From czhihua at vmware.com Fri Oct 13 11:50:00 2017 From: czhihua at vmware.com (Zhihua Cao) Date: Fri, 13 Oct 2017 04:50:00 -0700 Subject: [PATCH] Make ssl upstream server name check configurable Message-ID: <392a4360b5396af56bab.1507895400@ubuntu> # HG changeset patch # User Zhihua Cao # Date 1507889088 25200 # Fri Oct 13 03:04:48 2017 -0700 # Node ID 392a4360b5396af56bab63e63244ea5622f69972 # Parent 648b1cca8f50d83eea02a6cc2c105ae95a3f3d72 Make ssl upstream server name check configurable Now when nginx always check common name in upstream's certificate with upstream.ssl_name. But they are not always same, it check fails, ssl handshake will fail. If proxy_ssl_server_name_check is off, turn off the check. diff -r 648b1cca8f50 -r 392a4360b539 src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c Wed Oct 11 01:23:29 2017 -0700 +++ b/src/http/modules/ngx_http_proxy_module.c Fri Oct 13 03:04:48 2017 -0700 @@ -673,6 +673,13 @@ offsetof(ngx_http_proxy_loc_conf_t, upstream.ssl_verify), NULL }, + { ngx_string("proxy_ssl_server_name_check"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, upstream.ssl_server_name_check), + NULL }, + { ngx_string("proxy_ssl_verify_depth"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, @@ -2906,6 +2913,7 @@ conf->upstream.ssl_session_reuse = NGX_CONF_UNSET; conf->upstream.ssl_server_name = NGX_CONF_UNSET; conf->upstream.ssl_verify = NGX_CONF_UNSET; + conf->upstream.ssl_server_name_check = NGX_CONF_UNSET; conf->ssl_verify_depth = NGX_CONF_UNSET_UINT; conf->ssl_passwords = NGX_CONF_UNSET_PTR; #endif @@ -3237,6 +3245,8 @@ prev->upstream.ssl_server_name, 0); ngx_conf_merge_value(conf->upstream.ssl_verify, prev->upstream.ssl_verify, 0); + ngx_conf_merge_value(conf->upstream.ssl_server_name_check, + prev->upstream.ssl_server_name_check, 1); ngx_conf_merge_uint_value(conf->ssl_verify_depth, prev->ssl_verify_depth, 1); ngx_conf_merge_str_value(conf->ssl_trusted_certificate, diff -r 648b1cca8f50 -r 392a4360b539 src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c Wed Oct 11 01:23:29 2017 -0700 +++ b/src/http/ngx_http_upstream.c Fri Oct 13 03:04:48 2017 -0700 @@ -1733,7 +1733,8 @@ goto failed; } - if (ngx_ssl_check_host(c, &u->ssl_name) != NGX_OK) { + if (u->conf->ssl_server_name_check + && ngx_ssl_check_host(c, &u->ssl_name) != NGX_OK) { ngx_log_error(NGX_LOG_ERR, c->log, 0, "upstream SSL certificate does not match \"%V\"", &u->ssl_name); diff -r 648b1cca8f50 -r 392a4360b539 src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h Wed Oct 11 01:23:29 2017 -0700 +++ b/src/http/ngx_http_upstream.h Fri Oct 13 03:04:48 2017 -0700 @@ -229,6 +229,7 @@ ngx_http_complex_value_t *ssl_name; ngx_flag_t ssl_server_name; ngx_flag_t ssl_verify; + ngx_flag_t ssl_server_name_check; #endif ngx_str_t module; diff -r 648b1cca8f50 -r 392a4360b539 src/stream/ngx_stream_proxy_module.c --- a/src/stream/ngx_stream_proxy_module.c Wed Oct 11 01:23:29 2017 -0700 +++ b/src/stream/ngx_stream_proxy_module.c Fri Oct 13 03:04:48 2017 -0700 @@ -41,6 +41,7 @@ ngx_flag_t ssl_server_name; ngx_flag_t ssl_verify; + ngx_flag_t ssl_server_name_check; ngx_uint_t ssl_verify_depth; ngx_str_t ssl_trusted_certificate; ngx_str_t ssl_crl; @@ -270,6 +271,13 @@ offsetof(ngx_stream_proxy_srv_conf_t, ssl_verify), NULL }, + { ngx_string("proxy_ssl_server_name_check"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_proxy_srv_conf_t, ssl_server_name_check), + NULL }, + { ngx_string("proxy_ssl_verify_depth"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, @@ -1066,7 +1074,8 @@ u = s->upstream; - if (ngx_ssl_check_host(pc, &u->ssl_name) != NGX_OK) { + if (pscf->ssl_server_name_check + && ngx_ssl_check_host(pc, &u->ssl_name) != NGX_OK) { ngx_log_error(NGX_LOG_ERR, pc->log, 0, "upstream SSL certificate does not match \"%V\"", &u->ssl_name); @@ -1852,6 +1861,7 @@ conf->ssl_session_reuse = NGX_CONF_UNSET; conf->ssl_server_name = NGX_CONF_UNSET; conf->ssl_verify = NGX_CONF_UNSET; + conf->ssl_server_name_check = NGX_CONF_UNSET; conf->ssl_verify_depth = NGX_CONF_UNSET_UINT; conf->ssl_passwords = NGX_CONF_UNSET_PTR; #endif @@ -1917,6 +1927,8 @@ ngx_conf_merge_value(conf->ssl_verify, prev->ssl_verify, 0); + ngx_conf_merge_value(conf->ssl_server_name_check, prev->ssl_server_name_check, 1); + ngx_conf_merge_uint_value(conf->ssl_verify_depth, prev->ssl_verify_depth, 1); From czhihua at vmware.com Fri Oct 13 12:23:12 2017 From: czhihua at vmware.com (Zhihua Cao) Date: Fri, 13 Oct 2017 05:23:12 -0700 Subject: [PATCH] Make ssl upstream server name check configurable Message-ID: # HG changeset patch # User Zhihua Cao # Date 1507889088 25200 # Fri Oct 13 03:04:48 2017 -0700 # Node ID cef7fb3f127a2847b3898f8e71d4d445a4b81dd6 # Parent 648b1cca8f50d83eea02a6cc2c105ae95a3f3d72 Make ssl upstream server name check configurable Now nginx always check if the common name in the certificate sent from upstream. But they are not always same, if not same, ssl handshake will fail. Now make the check configurable, if proxy_ssl_server_name_check is off, turn off the check. The check is turned on by default. diff -r 648b1cca8f50 -r cef7fb3f127a src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c Wed Oct 11 01:23:29 2017 -0700 +++ b/src/http/modules/ngx_http_proxy_module.c Fri Oct 13 03:04:48 2017 -0700 @@ -673,6 +673,13 @@ offsetof(ngx_http_proxy_loc_conf_t, upstream.ssl_verify), NULL }, + { ngx_string("proxy_ssl_server_name_check"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, upstream.ssl_server_name_check), + NULL }, + { ngx_string("proxy_ssl_verify_depth"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, @@ -2906,6 +2913,7 @@ conf->upstream.ssl_session_reuse = NGX_CONF_UNSET; conf->upstream.ssl_server_name = NGX_CONF_UNSET; conf->upstream.ssl_verify = NGX_CONF_UNSET; + conf->upstream.ssl_server_name_check = NGX_CONF_UNSET; conf->ssl_verify_depth = NGX_CONF_UNSET_UINT; conf->ssl_passwords = NGX_CONF_UNSET_PTR; #endif @@ -3237,6 +3245,8 @@ prev->upstream.ssl_server_name, 0); ngx_conf_merge_value(conf->upstream.ssl_verify, prev->upstream.ssl_verify, 0); + ngx_conf_merge_value(conf->upstream.ssl_server_name_check, + prev->upstream.ssl_server_name_check, 1); ngx_conf_merge_uint_value(conf->ssl_verify_depth, prev->ssl_verify_depth, 1); ngx_conf_merge_str_value(conf->ssl_trusted_certificate, diff -r 648b1cca8f50 -r cef7fb3f127a src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c Wed Oct 11 01:23:29 2017 -0700 +++ b/src/http/ngx_http_upstream.c Fri Oct 13 03:04:48 2017 -0700 @@ -1733,7 +1733,8 @@ goto failed; } - if (ngx_ssl_check_host(c, &u->ssl_name) != NGX_OK) { + if (u->conf->ssl_server_name_check + && ngx_ssl_check_host(c, &u->ssl_name) != NGX_OK) { ngx_log_error(NGX_LOG_ERR, c->log, 0, "upstream SSL certificate does not match \"%V\"", &u->ssl_name); diff -r 648b1cca8f50 -r cef7fb3f127a src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h Wed Oct 11 01:23:29 2017 -0700 +++ b/src/http/ngx_http_upstream.h Fri Oct 13 03:04:48 2017 -0700 @@ -229,6 +229,7 @@ ngx_http_complex_value_t *ssl_name; ngx_flag_t ssl_server_name; ngx_flag_t ssl_verify; + ngx_flag_t ssl_server_name_check; #endif ngx_str_t module; diff -r 648b1cca8f50 -r cef7fb3f127a src/stream/ngx_stream_proxy_module.c --- a/src/stream/ngx_stream_proxy_module.c Wed Oct 11 01:23:29 2017 -0700 +++ b/src/stream/ngx_stream_proxy_module.c Fri Oct 13 03:04:48 2017 -0700 @@ -41,6 +41,7 @@ ngx_flag_t ssl_server_name; ngx_flag_t ssl_verify; + ngx_flag_t ssl_server_name_check; ngx_uint_t ssl_verify_depth; ngx_str_t ssl_trusted_certificate; ngx_str_t ssl_crl; @@ -270,6 +271,13 @@ offsetof(ngx_stream_proxy_srv_conf_t, ssl_verify), NULL }, + { ngx_string("proxy_ssl_server_name_check"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_proxy_srv_conf_t, ssl_server_name_check), + NULL }, + { ngx_string("proxy_ssl_verify_depth"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, @@ -1066,7 +1074,8 @@ u = s->upstream; - if (ngx_ssl_check_host(pc, &u->ssl_name) != NGX_OK) { + if (pscf->ssl_server_name_check + && ngx_ssl_check_host(pc, &u->ssl_name) != NGX_OK) { ngx_log_error(NGX_LOG_ERR, pc->log, 0, "upstream SSL certificate does not match \"%V\"", &u->ssl_name); @@ -1852,6 +1861,7 @@ conf->ssl_session_reuse = NGX_CONF_UNSET; conf->ssl_server_name = NGX_CONF_UNSET; conf->ssl_verify = NGX_CONF_UNSET; + conf->ssl_server_name_check = NGX_CONF_UNSET; conf->ssl_verify_depth = NGX_CONF_UNSET_UINT; conf->ssl_passwords = NGX_CONF_UNSET_PTR; #endif @@ -1917,6 +1927,8 @@ ngx_conf_merge_value(conf->ssl_verify, prev->ssl_verify, 0); + ngx_conf_merge_value(conf->ssl_server_name_check, prev->ssl_server_name_check, 1); + ngx_conf_merge_uint_value(conf->ssl_verify_depth, prev->ssl_verify_depth, 1); From mdounin at mdounin.ru Fri Oct 13 13:20:33 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Fri, 13 Oct 2017 16:20:33 +0300 Subject: [PATCH] Make ssl upstream server name check configurable In-Reply-To: References: Message-ID: <20171013132033.GN75166@mdounin.ru> Hello! On Fri, Oct 13, 2017 at 05:23:12AM -0700, Zhihua Cao wrote: > # HG changeset patch > # User Zhihua Cao > # Date 1507889088 25200 > # Fri Oct 13 03:04:48 2017 -0700 > # Node ID cef7fb3f127a2847b3898f8e71d4d445a4b81dd6 > # Parent 648b1cca8f50d83eea02a6cc2c105ae95a3f3d72 > Make ssl upstream server name check configurable > > Now nginx always check if the common name in the certificate sent > from upstream. But they are not always same, if not same, ssl > handshake will fail. > Now make the check configurable, if proxy_ssl_server_name_check is off, > turn off the check. > The check is turned on by default. The "proxy_ssl_name" directive can be used to adjust the name nginx asks for (if Server Name Indication is enabled) and verifies in the response, see http://nginx.org/r/proxy_ssl_name. If you think it is not enough, please explain the use case you are trying to use it for. (Also, please avoid replying to mailing list digests. It breaks threads and generally makes conversations very inconvenient.) [...] -- Maxim Dounin http://nginx.org/ From maulikpatelbtech at gmail.com Sun Oct 15 14:16:25 2017 From: maulikpatelbtech at gmail.com (Maulik patel) Date: Sun, 15 Oct 2017 19:46:25 +0530 Subject: nginx : PrettyUrl doent work (Yii2) Message-ID: Hi Team, You are doing Great Job. Just i need a small solution from you guys. I have installed nginx server and i set for backend 'enablePrettyUrl' => true, Below are the my default.conf file server { listen 80; server_name SERVER_IP; # note that these lines are originally from the "location /" block root /usr/share/nginx/html; index index.php index.html index.htm; location / { try_files $uri $uri/ =404; } error_page 404 /errors/404.html; error_page 500 502 503 504 /errors/50x.html; location = /errors/50x.html { root /usr/share/nginx/html; } location ~ \.php$ { try_files $uri =404; fastcgi_pass unix:/var/run/php-fpm/php-fpm.sock; fastcgi_index index.php; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; include fastcgi_params; }} When i run http://SERVER_IP/backend/web/CONTROLLER_NAME Its redirect to me in 404 error page. Can you please tell me how to enable pretty url in yii2 with nginx configuration? Please note that there is static website so my root path will be /usr/share/nginx/html Thanks, Maulik ------------ Your regards, Maulik Patel Phone : +91-942-935-9528 Skype : maulikpatelbtech LinkedIn : https://in.linkedin.com/in/maulikpatelbtech Stack : https://stackoverflow.com/users/833817 -------------- next part -------------- An HTML attachment was scrubbed... URL: From pluknet at nginx.com Mon Oct 16 10:24:11 2017 From: pluknet at nginx.com (Sergey Kandaurov) Date: Mon, 16 Oct 2017 10:24:11 +0000 Subject: [nginx] Postpone filter: handled ngx_http_postpone_filter_add() failures. Message-ID: details: http://hg.nginx.org/nginx/rev/d87f29d764ce branches: changeset: 7134:d87f29d764ce user: Sergey Kandaurov date: Mon Oct 16 13:13:25 2017 +0300 description: Postpone filter: handled ngx_http_postpone_filter_add() failures. In particular, if ngx_http_postpone_filter_add() fails in ngx_chain_add_copy(), the output chain of the postponed request was left in an invalid state. diffstat: src/http/ngx_http_postpone_filter_module.c | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diffs (26 lines): diff -r dc3b3cfd5d23 -r d87f29d764ce src/http/ngx_http_postpone_filter_module.c --- a/src/http/ngx_http_postpone_filter_module.c Fri Oct 13 00:32:26 2017 +0300 +++ b/src/http/ngx_http_postpone_filter_module.c Mon Oct 16 13:13:25 2017 +0300 @@ -63,7 +63,10 @@ ngx_http_postpone_filter(ngx_http_reques if (r != c->data) { if (in) { - ngx_http_postpone_filter_add(r, in); + if (ngx_http_postpone_filter_add(r, in) != NGX_OK) { + return NGX_ERROR; + } + return NGX_OK; } @@ -86,7 +89,9 @@ ngx_http_postpone_filter(ngx_http_reques } if (in) { - ngx_http_postpone_filter_add(r, in); + if (ngx_http_postpone_filter_add(r, in) != NGX_OK) { + return NGX_ERROR; + } } do { From mdounin at mdounin.ru Tue Oct 17 12:51:09 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 12:51:09 +0000 Subject: [nginx] Version bump. Message-ID: details: http://hg.nginx.org/nginx/rev/5e1112ac82d5 branches: stable-1.12 changeset: 7135:5e1112ac82d5 user: Maxim Dounin date: Fri Oct 13 20:23:57 2017 +0300 description: Version bump. diffstat: src/core/nginx.h | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff --git a/src/core/nginx.h b/src/core/nginx.h --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1012001 -#define NGINX_VERSION "1.12.1" +#define nginx_version 1012002 +#define NGINX_VERSION "1.12.2" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From mdounin at mdounin.ru Tue Oct 17 12:51:12 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 12:51:12 +0000 Subject: [nginx] Added missing "fall through" comments (ticket #1259). Message-ID: details: http://hg.nginx.org/nginx/rev/d409ab1e8e4d branches: stable-1.12 changeset: 7136:d409ab1e8e4d user: Maxim Dounin date: Thu Apr 27 16:57:18 2017 +0300 description: Added missing "fall through" comments (ticket #1259). Found by gcc7 (-Wimplicit-fallthrough). diffstat: src/core/ngx_murmurhash.c | 2 ++ src/http/ngx_http_parse.c | 3 +++ src/os/unix/ngx_process.c | 1 + 3 files changed, 6 insertions(+), 0 deletions(-) diffs (52 lines): diff --git a/src/core/ngx_murmurhash.c b/src/core/ngx_murmurhash.c --- a/src/core/ngx_murmurhash.c +++ b/src/core/ngx_murmurhash.c @@ -35,8 +35,10 @@ ngx_murmur_hash2(u_char *data, size_t le switch (len) { case 3: h ^= data[2] << 16; + /* fall through */ case 2: h ^= data[1] << 8; + /* fall through */ case 1: h ^= data[0]; h *= 0x5bd1e995; diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c --- a/src/http/ngx_http_parse.c +++ b/src/http/ngx_http_parse.c @@ -1390,6 +1390,7 @@ ngx_http_parse_complex_uri(ngx_http_requ goto done; case '+': r->plus_in_uri = 1; + /* fall through */ default: state = sw_usual; *u++ = ch; @@ -1431,6 +1432,7 @@ ngx_http_parse_complex_uri(ngx_http_requ goto done; case '+': r->plus_in_uri = 1; + /* fall through */ default: state = sw_usual; *u++ = ch; @@ -1478,6 +1480,7 @@ ngx_http_parse_complex_uri(ngx_http_requ goto done; case '+': r->plus_in_uri = 1; + /* fall through */ default: state = sw_usual; *u++ = ch; diff --git a/src/os/unix/ngx_process.c b/src/os/unix/ngx_process.c --- a/src/os/unix/ngx_process.c +++ b/src/os/unix/ngx_process.c @@ -405,6 +405,7 @@ ngx_signal_handler(int signo) break; } ngx_debug_quit = 1; + /* fall through */ case ngx_signal_value(NGX_SHUTDOWN_SIGNAL): ngx_quit = 1; action = ", shutting down"; From mdounin at mdounin.ru Tue Oct 17 13:06:40 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:06:40 +0000 Subject: [nginx] Fixed deferred accept with EPOLLRDHUP enabled (ticket #1278). Message-ID: details: http://hg.nginx.org/nginx/rev/0e05b35beebf branches: stable-1.12 changeset: 7137:0e05b35beebf user: Roman Arutyunyan date: Wed May 24 13:17:08 2017 +0300 description: Fixed deferred accept with EPOLLRDHUP enabled (ticket #1278). Previously, the read event of the accepted connection was marked ready, but not available. This made EPOLLRDHUP-related code (for example, in ngx_unix_recv()) expect more data from the socket, leading to unexpected behavior. For example, if SSL, PROXY protocol and deferred accept were enabled on a listen socket, the client connection was aborted due to unexpected return value of c->recv(). diffstat: src/event/ngx_event_accept.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c --- a/src/event/ngx_event_accept.c +++ b/src/event/ngx_event_accept.c @@ -238,7 +238,7 @@ ngx_event_accept(ngx_event_t *ev) if (ev->deferred_accept) { rev->ready = 1; -#if (NGX_HAVE_KQUEUE) +#if (NGX_HAVE_KQUEUE || NGX_HAVE_EPOLLRDHUP) rev->available = 1; #endif } From mdounin at mdounin.ru Tue Oct 17 13:06:42 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:06:42 +0000 Subject: [nginx] Core: disabled SO_REUSEPORT when testing config (ticket #1300). Message-ID: details: http://hg.nginx.org/nginx/rev/05bd1baabf87 branches: stable-1.12 changeset: 7138:05bd1baabf87 user: Maxim Dounin date: Tue Jul 11 19:59:56 2017 +0300 description: Core: disabled SO_REUSEPORT when testing config (ticket #1300). When closing a socket with SO_REUSEPORT, Linux drops all connections waiting in this socket's listen queue. Previously, it was believed to only result in connection resets when reconfiguring nginx to use smaller number of worker processes. It also results in connection resets during configuration testing though. Workaround is to avoid using SO_REUSEPORT when testing configuration. It should prevent listening sockets from being created if a conflicting socket already exists, while still preserving detection of other possible errors. It should also cover UDP sockets. The only downside of this approach seems to be that a configuration testing won't be able to properly report the case when nginx was compiled with SO_REUSEPORT, but the kernel is not able to set it. Such errors will be reported on a real start instead. diffstat: src/core/ngx_connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -473,7 +473,7 @@ ngx_open_listening_sockets(ngx_cycle_t * #if (NGX_HAVE_REUSEPORT) - if (ls[i].reuseport) { + if (ls[i].reuseport && !ngx_test_config) { int reuseport; reuseport = 1; From mdounin at mdounin.ru Tue Oct 17 13:06:45 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:06:45 +0000 Subject: [nginx] Core: fixed error message on setsockopt(SO_REUSEPORT) failure. Message-ID: details: http://hg.nginx.org/nginx/rev/b56ddcce7513 branches: stable-1.12 changeset: 7139:b56ddcce7513 user: Maxim Dounin date: Tue Jul 11 20:06:52 2017 +0300 description: Core: fixed error message on setsockopt(SO_REUSEPORT) failure. The error is fatal when configuring a new socket, so the ", ignored" part is not appropriate and was removed. diffstat: src/core/ngx_connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -483,7 +483,7 @@ ngx_open_listening_sockets(ngx_cycle_t * == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, - "setsockopt(SO_REUSEPORT) %V failed, ignored", + "setsockopt(SO_REUSEPORT) %V failed", &ls[i].addr_text); if (ngx_close_socket(s) == -1) { From mdounin at mdounin.ru Tue Oct 17 13:06:47 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:06:47 +0000 Subject: [nginx] Updated PCRE used for win32 builds. Message-ID: details: http://hg.nginx.org/nginx/rev/90ef2b838c61 branches: stable-1.12 changeset: 7140:90ef2b838c61 user: Maxim Dounin date: Tue Aug 08 15:21:10 2017 +0300 description: Updated PCRE used for win32 builds. diffstat: misc/GNUmakefile | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff --git a/misc/GNUmakefile b/misc/GNUmakefile --- a/misc/GNUmakefile +++ b/misc/GNUmakefile @@ -8,7 +8,7 @@ CC = cl OBJS = objs.msvc8 OPENSSL = openssl-1.0.2l ZLIB = zlib-1.2.11 -PCRE = pcre-8.40 +PCRE = pcre-8.41 release: export From mdounin at mdounin.ru Tue Oct 17 13:06:50 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:06:50 +0000 Subject: [nginx] Range filter: changed type for total length to off_t. Message-ID: details: http://hg.nginx.org/nginx/rev/dfa0250bf222 branches: stable-1.12 changeset: 7141:dfa0250bf222 user: Maxim Dounin date: Thu Aug 10 22:21:23 2017 +0300 description: Range filter: changed type for total length to off_t. Total length of a response with multiple ranges can be larger than a size_t variable can hold, so type changed to off_t. Previously, an incorrect Content-Length was returned when requesting more than 4G of ranges from a large enough file on a 32-bit system. An additional size_t variable introduced to calculate size of the boundary header buffer, as off_t is not needed here and will require type casts on win32. Reported by Shuxin Yang, http://mailman.nginx.org/pipermail/nginx/2017-July/054384.html. diffstat: src/http/modules/ngx_http_range_filter_module.c | 17 +++++++++-------- 1 files changed, 9 insertions(+), 8 deletions(-) diffs (44 lines): diff --git a/src/http/modules/ngx_http_range_filter_module.c b/src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c +++ b/src/http/modules/ngx_http_range_filter_module.c @@ -458,23 +458,24 @@ static ngx_int_t ngx_http_range_multipart_header(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx) { - size_t len; + off_t len; + size_t size; ngx_uint_t i; ngx_http_range_t *range; ngx_atomic_uint_t boundary; - len = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN - + sizeof(CRLF "Content-Type: ") - 1 - + r->headers_out.content_type.len - + sizeof(CRLF "Content-Range: bytes ") - 1; + size = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + + sizeof(CRLF "Content-Type: ") - 1 + + r->headers_out.content_type.len + + sizeof(CRLF "Content-Range: bytes ") - 1; if (r->headers_out.content_type_len == r->headers_out.content_type.len && r->headers_out.charset.len) { - len += sizeof("; charset=") - 1 + r->headers_out.charset.len; + size += sizeof("; charset=") - 1 + r->headers_out.charset.len; } - ctx->boundary_header.data = ngx_pnalloc(r->pool, len); + ctx->boundary_header.data = ngx_pnalloc(r->pool, size); if (ctx->boundary_header.data == NULL) { return NGX_ERROR; } @@ -564,7 +565,7 @@ ngx_http_range_multipart_header(ngx_http - range[i].content_range.data; len += ctx->boundary_header.len + range[i].content_range.len - + (size_t) (range[i].end - range[i].start); + + (range[i].end - range[i].start); } r->headers_out.content_length_n = len; From mdounin at mdounin.ru Tue Oct 17 13:06:52 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:06:52 +0000 Subject: [nginx] Stream: relaxed next upstream condition (ticket #1317). Message-ID: details: http://hg.nginx.org/nginx/rev/b9d919b53593 branches: stable-1.12 changeset: 7142:b9d919b53593 user: Roman Arutyunyan date: Mon Sep 11 15:32:31 2017 +0300 description: Stream: relaxed next upstream condition (ticket #1317). When switching to a next upstream, some buffers could be stuck in the middle of the filter chain. A condition existed that raised an error when this happened. As it turned out, this condition prevented switching to a next upstream if ssl preread was used with the TCP protocol (see the ticket). In fact, the condition does not make sense for TCP, since after successful connection to an upstream switching to another upstream never happens. As for UDP, the issue with stuck buffers is unlikely to happen, but is still possible. Specifically, if a filter delays sending data to upstream. The condition can be relaxed to only check the "buffered" bitmask of the upstream connection. The new condition is simpler and fixes the ticket issue as well. Additionally, the upstream_out chain is now reset for UDP prior to connecting to a new upstream to prevent repeating the client data twice. diffstat: src/stream/ngx_stream_proxy_module.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (23 lines): diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c --- a/src/stream/ngx_stream_proxy_module.c +++ b/src/stream/ngx_stream_proxy_module.c @@ -1677,13 +1677,17 @@ ngx_stream_proxy_next_upstream(ngx_strea u = s->upstream; pc = u->peer.connection; - if (u->upstream_out || u->upstream_busy || (pc && pc->buffered)) { + if (pc && pc->buffered) { ngx_log_error(NGX_LOG_ERR, s->connection->log, 0, - "pending buffers on next upstream"); + "buffered data on next upstream"); ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR); return; } + if (s->connection->type == SOCK_DGRAM) { + u->upstream_out = NULL; + } + if (u->peer.sockaddr) { u->peer.free(&u->peer, u->peer.data, NGX_PEER_FAILED); u->peer.sockaddr = NULL; From mdounin at mdounin.ru Tue Oct 17 13:06:55 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:06:55 +0000 Subject: [nginx] HTTP/2: enforce writing the sync request body buffer to file. Message-ID: details: http://hg.nginx.org/nginx/rev/e532d397ca5e branches: stable-1.12 changeset: 7143:e532d397ca5e user: Valentin Bartenev date: Wed Oct 04 21:15:15 2017 +0300 description: HTTP/2: enforce writing the sync request body buffer to file. The sync flag of HTTP/2 request body buffer is used when the size of request body is unknown or bigger than configured "client_body_buffer_size". In this case the buffer points to body data inside the global receive buffer that is used for reading all HTTP/2 connections in the worker process. Thus, when the sync flag is set, the buffer must be flushed to a temporary file, otherwise the request body data can be overwritten. Previously, the sync buffer wasn't flushed to a temporary file if the whole body was received in one DATA frame with the END_STREAM flag and wasn't copied into the HTTP/2 body preread buffer. As a result, the request body might be corrupted (ticket #1384). Now, setting r->request_body_in_file_only enforces writing the sync buffer to a temporary file in all cases. diffstat: src/http/v2/ngx_http_v2.c | 7 ++----- 1 files changed, 2 insertions(+), 5 deletions(-) diffs (24 lines): diff --git a/src/http/v2/ngx_http_v2.c b/src/http/v2/ngx_http_v2.c --- a/src/http/v2/ngx_http_v2.c +++ b/src/http/v2/ngx_http_v2.c @@ -3557,11 +3557,6 @@ ngx_http_v2_read_request_body(ngx_http_r rb->buf = ngx_create_temp_buf(r->pool, (size_t) len); } else { - if (stream->preread) { - /* enforce writing preread buffer to file */ - r->request_body_in_file_only = 1; - } - rb->buf = ngx_calloc_buf(r->pool); if (rb->buf != NULL) { @@ -3660,6 +3655,8 @@ ngx_http_v2_process_request_body(ngx_htt buf->pos = buf->start = pos; buf->last = buf->end = pos + size; + r->request_body_in_file_only = 1; + } else { if (size > (size_t) (buf->end - buf->last)) { ngx_log_error(NGX_LOG_INFO, fc->log, 0, From mdounin at mdounin.ru Tue Oct 17 13:06:57 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:06:57 +0000 Subject: [nginx] Fixed the NGX_UNIX_ADDRSTRLEN macro. Message-ID: details: http://hg.nginx.org/nginx/rev/dea142ac3deb branches: stable-1.12 changeset: 7144:dea142ac3deb user: Ruslan Ermilov date: Mon Sep 25 15:19:24 2017 +0300 description: Fixed the NGX_UNIX_ADDRSTRLEN macro. diffstat: src/core/ngx_inet.h | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (17 lines): diff --git a/src/core/ngx_inet.h b/src/core/ngx_inet.h --- a/src/core/ngx_inet.h +++ b/src/core/ngx_inet.h @@ -17,10 +17,11 @@ #define NGX_INET6_ADDRSTRLEN \ (sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255") - 1) #define NGX_UNIX_ADDRSTRLEN \ - (sizeof(struct sockaddr_un) - offsetof(struct sockaddr_un, sun_path)) + (sizeof("unix:") - 1 + \ + sizeof(struct sockaddr_un) - offsetof(struct sockaddr_un, sun_path)) #if (NGX_HAVE_UNIX_DOMAIN) -#define NGX_SOCKADDR_STRLEN (sizeof("unix:") - 1 + NGX_UNIX_ADDRSTRLEN) +#define NGX_SOCKADDR_STRLEN NGX_UNIX_ADDRSTRLEN #elif (NGX_HAVE_INET6) #define NGX_SOCKADDR_STRLEN (NGX_INET6_ADDRSTRLEN + sizeof("[]:65535") - 1) #else From mdounin at mdounin.ru Tue Oct 17 13:06:59 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:06:59 +0000 Subject: [nginx] Fixed buffer overread with unix sockets after accept(). Message-ID: details: http://hg.nginx.org/nginx/rev/83600ce25995 branches: stable-1.12 changeset: 7145:83600ce25995 user: Maxim Dounin date: Wed Oct 04 21:19:33 2017 +0300 description: Fixed buffer overread with unix sockets after accept(). Some OSes (notably macOS, NetBSD, and Solaris) allow unix socket addresses larger than struct sockaddr_un. Moreover, some of them (macOS, Solaris) return socklen of the socket address before it was truncated to fit the buffer provided. As such, on these systems socklen must not be used without additional check that it is within the buffer provided. Appropriate checks added to ngx_event_accept() (after accept()), ngx_event_recvmsg() (after recvmsg()), and ngx_set_inherited_sockets() (after getsockname()). We also obtain socket addresses via getsockname() in ngx_connection_local_sockaddr(), but it does not need any checks as it is only used for INET and INET6 sockets (as there can be no wildcard unix sockets). diffstat: src/core/ngx_connection.c | 4 ++++ src/event/ngx_event_accept.c | 8 ++++++++ 2 files changed, 12 insertions(+), 0 deletions(-) diffs (39 lines): diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -165,6 +165,10 @@ ngx_set_inherited_sockets(ngx_cycle_t *c continue; } + if (ls[i].socklen > (socklen_t) sizeof(ngx_sockaddr_t)) { + ls[i].socklen = sizeof(ngx_sockaddr_t); + } + switch (ls[i].sockaddr->sa_family) { #if (NGX_HAVE_INET6) diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c --- a/src/event/ngx_event_accept.c +++ b/src/event/ngx_event_accept.c @@ -164,6 +164,10 @@ ngx_event_accept(ngx_event_t *ev) return; } + if (socklen > (socklen_t) sizeof(ngx_sockaddr_t)) { + socklen = sizeof(ngx_sockaddr_t); + } + c->sockaddr = ngx_palloc(c->pool, socklen); if (c->sockaddr == NULL) { ngx_close_accepted_connection(c); @@ -440,6 +444,10 @@ ngx_event_recvmsg(ngx_event_t *ev) c->type = SOCK_DGRAM; c->socklen = msg.msg_namelen; + if (c->socklen > (socklen_t) sizeof(ngx_sockaddr_t)) { + c->socklen = sizeof(ngx_sockaddr_t); + } + #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_active, 1); #endif From mdounin at mdounin.ru Tue Oct 17 13:07:02 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:07:02 +0000 Subject: [nginx] Fixed handling of non-null-terminated unix sockets. Message-ID: details: http://hg.nginx.org/nginx/rev/5c25f01bbd52 branches: stable-1.12 changeset: 7146:5c25f01bbd52 user: Maxim Dounin date: Wed Oct 04 21:19:38 2017 +0300 description: Fixed handling of non-null-terminated unix sockets. At least FreeBSD, macOS, NetBSD, and OpenBSD can return unix sockets with non-null-terminated sun_path. Additionally, the address may become non-null-terminated if it does not fit into the buffer provided and was truncated (may happen on macOS, NetBSD, and Solaris, which allow unix socket addresess larger than struct sockaddr_un). As such, ngx_sock_ntop() might overread the sockaddr provided, as it used "%s" format and thus assumed null-terminated string. To fix this, the ngx_strnlen() function was introduced, and it is now used to calculate correct length of sun_path. diffstat: src/core/ngx_inet.c | 4 +++- src/core/ngx_string.c | 16 ++++++++++++++++ src/core/ngx_string.h | 2 ++ 3 files changed, 21 insertions(+), 1 deletions(-) diffs (52 lines): diff --git a/src/core/ngx_inet.c b/src/core/ngx_inet.c --- a/src/core/ngx_inet.c +++ b/src/core/ngx_inet.c @@ -241,7 +241,9 @@ ngx_sock_ntop(struct sockaddr *sa, sockl p = ngx_snprintf(text, len, "unix:%Z"); } else { - p = ngx_snprintf(text, len, "unix:%s%Z", saun->sun_path); + n = ngx_strnlen((u_char *) saun->sun_path, + socklen - offsetof(struct sockaddr_un, sun_path)); + p = ngx_snprintf(text, len, "unix:%*s%Z", n, saun->sun_path); } /* we do not include trailing zero in address length */ diff --git a/src/core/ngx_string.c b/src/core/ngx_string.c --- a/src/core/ngx_string.c +++ b/src/core/ngx_string.c @@ -29,6 +29,22 @@ ngx_strlow(u_char *dst, u_char *src, siz } +size_t +ngx_strnlen(u_char *p, size_t n) +{ + size_t i; + + for (i = 0; i < n; i++) { + + if (p[i] == '\0') { + return i; + } + } + + return n; +} + + u_char * ngx_cpystrn(u_char *dst, u_char *src, size_t n) { diff --git a/src/core/ngx_string.h b/src/core/ngx_string.h --- a/src/core/ngx_string.h +++ b/src/core/ngx_string.h @@ -60,6 +60,8 @@ void ngx_strlow(u_char *dst, u_char *src #define ngx_strstr(s1, s2) strstr((const char *) s1, (const char *) s2) #define ngx_strlen(s) strlen((const char *) s) +size_t ngx_strnlen(u_char *p, size_t n); + #define ngx_strchr(s1, c) strchr((const char *) s1, (int) c) static ngx_inline u_char * From mdounin at mdounin.ru Tue Oct 17 13:07:04 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:07:04 +0000 Subject: [nginx] Fixed handling of unix sockets in $binary_remote_addr. Message-ID: details: http://hg.nginx.org/nginx/rev/0800444254e8 branches: stable-1.12 changeset: 7147:0800444254e8 user: Maxim Dounin date: Wed Oct 04 21:19:42 2017 +0300 description: Fixed handling of unix sockets in $binary_remote_addr. Previously, unix sockets were treated as AF_INET ones, and this may result in buffer overread on Linux, where unbound unix sockets have 2-byte addresses. Note that it is not correct to use just sun_path as a binary representation for unix sockets. This will result in an empty string for unbound unix sockets, and thus behaviour of limit_req and limit_conn will change when switching from $remote_addr to $binary_remote_addr. As such, normal text representation is used. Reported by Stephan Dollberg. diffstat: src/http/ngx_http_variables.c | 12 ++++++++++++ src/stream/ngx_stream_variables.c | 12 ++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-) diffs (44 lines): diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c --- a/src/http/ngx_http_variables.c +++ b/src/http/ngx_http_variables.c @@ -1225,6 +1225,18 @@ ngx_http_variable_binary_remote_addr(ngx break; #endif +#if (NGX_HAVE_UNIX_DOMAIN) + case AF_UNIX: + + v->len = r->connection->addr_text.len; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = r->connection->addr_text.data; + + break; +#endif + default: /* AF_INET */ sin = (struct sockaddr_in *) r->connection->sockaddr; diff --git a/src/stream/ngx_stream_variables.c b/src/stream/ngx_stream_variables.c --- a/src/stream/ngx_stream_variables.c +++ b/src/stream/ngx_stream_variables.c @@ -481,6 +481,18 @@ ngx_stream_variable_binary_remote_addr(n break; #endif +#if (NGX_HAVE_UNIX_DOMAIN) + case AF_UNIX: + + v->len = s->connection->addr_text.len; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = s->connection->addr_text.data; + + break; +#endif + default: /* AF_INET */ sin = (struct sockaddr_in *) s->connection->sockaddr; From mdounin at mdounin.ru Tue Oct 17 13:07:07 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:07:07 +0000 Subject: [nginx] Fixed build without IPv6, broken by 874171c3c71a. Message-ID: details: http://hg.nginx.org/nginx/rev/bc902ff79de8 branches: stable-1.12 changeset: 7148:bc902ff79de8 user: Maxim Dounin date: Thu Oct 05 16:50:35 2017 +0300 description: Fixed build without IPv6, broken by 874171c3c71a. diffstat: src/core/ngx_inet.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (16 lines): diff --git a/src/core/ngx_inet.c b/src/core/ngx_inet.c --- a/src/core/ngx_inet.c +++ b/src/core/ngx_inet.c @@ -182,9 +182,11 @@ ngx_sock_ntop(struct sockaddr *sa, sockl ngx_uint_t port) { u_char *p; +#if (NGX_HAVE_INET6 || NGX_HAVE_UNIX_DOMAIN) + size_t n; +#endif struct sockaddr_in *sin; #if (NGX_HAVE_INET6) - size_t n; struct sockaddr_in6 *sin6; #endif #if (NGX_HAVE_UNIX_DOMAIN) From mdounin at mdounin.ru Tue Oct 17 13:21:14 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:21:14 +0000 Subject: [nginx] nginx-1.12.2-RELEASE Message-ID: details: http://hg.nginx.org/nginx/rev/c0c5f8b806fe branches: stable-1.12 changeset: 7149:c0c5f8b806fe user: Maxim Dounin date: Tue Oct 17 16:16:37 2017 +0300 description: nginx-1.12.2-RELEASE diffstat: docs/xml/nginx/changes.xml | 69 ++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 69 insertions(+), 0 deletions(-) diffs (79 lines): diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,75 @@ + + + + +?????????? SSL-?????????? ????? ???????????, ???? ????????????? +?????????? accept ? ???????? proxy_protocol ????????? listen. + + +client SSL connections were immediately closed if deferred accept +and the "proxy_protocol" parameter of the "listen" directive were used. + + + + + +?????????? ?????????? ????? ???????????? ??? ???????????? ????????????, +???? ????????????? ???????? reuseport ????????? listen ?? Linux. + + +client connections might be dropped during configuration testing +when using the "reuseport" parameter of the "listen" directive on Linux. + + + + + +?? 32-?????? ?????????? +??? ??????? ????? 4 ???????? ? ??????? ?????????? ?????????? +???????????? ???????????? ????? ??????. + + +incorrect response length was returned +on 32-bit platforms when requesting more than 4 gigabytes +with multiple ranges. + + + + + +??? ????????????? ????????? ssl_preread +? ?????? stream ?? ???????? ???????????? ?? ????????? ??????. + + +switching to the next upstream server in the stream module did not work +when using the "ssl_preread" directive. + + + + + +??? ????????????? HTTP/2 ???? ??????? ????? ???? ??????????. + + +when using HTTP/2 client request body might be corrupted. + + + + + +? ????????? ??????? ???????? ??? ????????????? unix domain ???????. + + +in handling of client addresses when using unix domain sockets. + + + + + + From mdounin at mdounin.ru Tue Oct 17 13:21:16 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 13:21:16 +0000 Subject: [nginx] release-1.12.2 tag Message-ID: details: http://hg.nginx.org/nginx/rev/d00a24dd3dee branches: stable-1.12 changeset: 7150:d00a24dd3dee user: Maxim Dounin date: Tue Oct 17 16:16:37 2017 +0300 description: release-1.12.2 tag diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -414,3 +414,4 @@ 7f394e433f0003222aa6531931ecc0b24740d5e4 3d0e8655f897959e48cc74e87670bb5492a58871 release-1.11.13 e265d962841f90f1fa148c8f4eed4a4d3a9b0208 release-1.12.0 08b443b6b59dbe5d02354047c649a00c40e7b748 release-1.12.1 +c0c5f8b806fe2b8a6a059447c712376293d6ccde release-1.12.2 From mdounin at mdounin.ru Tue Oct 17 17:58:38 2017 From: mdounin at mdounin.ru (Maxim Dounin) Date: Tue, 17 Oct 2017 17:58:38 +0000 Subject: [nginx] Core: free shared memory zones only after reconfiguration. Message-ID: details: http://hg.nginx.org/nginx/rev/9ef704d8563a branches: changeset: 7151:9ef704d8563a user: Maxim Dounin date: Tue Oct 17 19:52:16 2017 +0300 description: Core: free shared memory zones only after reconfiguration. This is what usually happens for zones no longer used in the new configuration, but zones where size or tag were changed were freed when creating new memory zones. If reconfiguration failed (for example, due to a conflicting listening socket), this resulted in a segmentation fault in the master process. Reported by Zhihua Cao, http://mailman.nginx.org/pipermail/nginx-devel/2017-October/010536.html. diffstat: src/core/ngx_cycle.c | 24 +++++++++++++++++------- 1 files changed, 17 insertions(+), 7 deletions(-) diffs (44 lines): diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c --- a/src/core/ngx_cycle.c +++ b/src/core/ngx_cycle.c @@ -470,8 +470,6 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) goto shm_zone_found; } - ngx_shm_free(&oshm_zone[n].shm); - break; } @@ -662,14 +660,26 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) n = 0; } - if (oshm_zone[i].shm.name.len == shm_zone[n].shm.name.len - && ngx_strncmp(oshm_zone[i].shm.name.data, - shm_zone[n].shm.name.data, - oshm_zone[i].shm.name.len) - == 0) + if (oshm_zone[i].shm.name.len != shm_zone[n].shm.name.len) { + continue; + } + + if (ngx_strncmp(oshm_zone[i].shm.name.data, + shm_zone[n].shm.name.data, + oshm_zone[i].shm.name.len) + != 0) + { + continue; + } + + if (oshm_zone[i].tag == shm_zone[n].tag + && oshm_zone[i].shm.size == shm_zone[n].shm.size + && !oshm_zone[i].noreuse) { goto live_shm_zone; } + + break; } ngx_shm_free(&oshm_zone[i].shm); From vbart at nginx.com Tue Oct 17 18:31:55 2017 From: vbart at nginx.com (Valentin V. Bartenev) Date: Tue, 17 Oct 2017 21:31:55 +0300 Subject: nginx : PrettyUrl doent work (Yii2) In-Reply-To: References: Message-ID: <1547169.985k0kWOpU@vbart-workstation> On Sunday 15 October 2017 19:46:25 Maulik patel wrote: [..] > > Can you please tell me how to enable pretty url in yii2 with nginx > configuration? Please note that there is static website so my root path > will be /usr/share/nginx/html > This is nginx developers mailing list. It's intended only for questions about nginx development and for code contributions. Please ask your question in the users mailing list: http://mailman.nginx.org/mailman/listinfo/nginx wbr. Valentin V. Bartenev From oleh.havryliuk at ria.com Fri Oct 20 13:48:34 2017 From: oleh.havryliuk at ria.com (=?UTF-8?B?0J7Qu9C10LMg0JPQsNCy0YDQuNC70Y7Qug==?=) Date: Fri, 20 Oct 2017 16:48:34 +0300 Subject: http_browser_module: added Google Chrome support Message-ID: -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: ngx_http_browser_module.patch Type: text/x-patch Size: 1754 bytes Desc: not available URL: From hucong.c at foxmail.com Thu Oct 26 18:23:47 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 02:23:47 +0800 Subject: [patch] SSI: fixed type. Message-ID: # HG changeset patch # User hucongcong # Date 1509035438 -28800 # Fri Oct 27 00:30:38 2017 +0800 # Node ID 0dc91aea7cd6e8398872ac3615ce1294b06e80af # Parent 9ef704d8563af4aff6817ab1c694fb40591f20b3 SSI: fixed type. diff -r 9ef704d8563a -r 0dc91aea7cd6 src/http/modules/ngx_http_ssi_filter_module.c --- a/src/http/modules/ngx_http_ssi_filter_module.c Tue Oct 17 19:52:16 2017 +0300 +++ b/src/http/modules/ngx_http_ssi_filter_module.c Fri Oct 27 00:30:38 2017 +0800 @@ -1630,8 +1630,7 @@ ngx_http_ssi_evaluate_string(ngx_http_re u_char ch, *p, **value, *data, *part_data; size_t *size, len, prefix, part_len; ngx_str_t var, *val; - ngx_int_t key; - ngx_uint_t i, n, bracket, quoted; + ngx_uint_t i, n, bracket, quoted, key; ngx_array_t lengths, values; ngx_http_variable_value_t *vv; @@ -1883,9 +1882,8 @@ ngx_http_ssi_regex_match(ngx_http_reques int rc, *captures; u_char *p, errstr[NGX_MAX_CONF_ERRSTR]; size_t size; - ngx_int_t key; ngx_str_t *vv, name, value; - ngx_uint_t i, n; + ngx_uint_t i, n, key; ngx_http_ssi_ctx_t *ctx; ngx_http_ssi_var_t *var; ngx_regex_compile_t rgc; @@ -1988,10 +1986,10 @@ static ngx_int_t ngx_http_ssi_include(ngx_http_request_t *r, ngx_http_ssi_ctx_t *ctx, ngx_str_t **params) { - ngx_int_t rc, key; + ngx_int_t rc; ngx_str_t *uri, *file, *wait, *set, *stub, args; ngx_buf_t *b; - ngx_uint_t flags, i; + ngx_uint_t flags, i, key; ngx_chain_t *cl, *tl, **ll, *out; ngx_http_request_t *sr; ngx_http_ssi_var_t *var; @@ -2248,9 +2246,9 @@ ngx_http_ssi_echo(ngx_http_request_t *r, { u_char *p; uintptr_t len; - ngx_int_t key; ngx_buf_t *b; ngx_str_t *var, *value, *enc, text; + ngx_uint_t key; ngx_chain_t *cl; ngx_http_variable_value_t *vv; @@ -2410,8 +2408,9 @@ static ngx_int_t ngx_http_ssi_set(ngx_http_request_t *r, ngx_http_ssi_ctx_t *ctx, ngx_str_t **params) { - ngx_int_t key, rc; + ngx_int_t rc; ngx_str_t *name, *value, *vv; + ngx_uint_t key; ngx_http_ssi_var_t *var; ngx_http_ssi_ctx_t *mctx; From hucong.c at foxmail.com Thu Oct 26 18:24:59 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 02:24:59 +0800 Subject: [patch] Range filter: more appropriate restriction on max ranges. Message-ID: # HG changeset patch # User hucongcong # Date 1509035936 -28800 # Fri Oct 27 00:38:56 2017 +0800 # Node ID a429a3b21a3950d69781e109111831e7c5f33566 # Parent 9ef704d8563af4aff6817ab1c694fb40591f20b3 Range filter: more appropriate restriction on max ranges. diff -r 9ef704d8563a -r a429a3b21a39 src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c Tue Oct 17 19:52:16 2017 +0300 +++ b/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 00:38:56 2017 +0800 @@ -369,6 +369,11 @@ ngx_http_range_parse(ngx_http_request_t found: if (start < end) { + + if (ranges-- == 0) { + return NGX_DECLINED; + } + range = ngx_array_push(&ctx->ranges); if (range == NULL) { return NGX_ERROR; @@ -383,10 +388,6 @@ ngx_http_range_parse(ngx_http_request_t size += end - start; - if (ranges-- == 0) { - return NGX_DECLINED; - } - } else if (start == 0) { return NGX_DECLINED; } From hucong.c at foxmail.com Thu Oct 26 18:26:17 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 02:26:17 +0800 Subject: [patch] Range filter: support multiple ranges. Message-ID: # HG changeset patch # User hucongcong # Date 1509038506 -28800 # Fri Oct 27 01:21:46 2017 +0800 # Node ID 798418b3f9c21811debf6cbeaea3eb554c886905 # Parent a429a3b21a3950d69781e109111831e7c5f33566 Range filter: support multiple ranges. Nginx will return 416 when the ranges (first-byte-pos) are not listed in ascending order or when one byte-range-spec contains another. And besides, nginx will coalesce any of the ranges that overlap, or that are separated by a gap that is smaller than the NGX_HTTP_RANGE_MULTIPART_GAP macro. diff -r a429a3b21a39 -r 798418b3f9c2 src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 00:38:56 2017 +0800 +++ b/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 01:21:46 2017 +0800 @@ -10,6 +10,9 @@ #include +#define NGX_HTTP_RANGE_MULTIPART_GAP 80 + + /* * the single part format: * @@ -55,6 +58,7 @@ typedef struct { typedef struct { off_t offset; ngx_str_t boundary_header; + ngx_uint_t index; /* start with 1 */ ngx_array_t ranges; } ngx_http_range_filter_ctx_t; @@ -66,12 +70,14 @@ static ngx_int_t ngx_http_range_singlepa static ngx_int_t ngx_http_range_multipart_header(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx); static ngx_int_t ngx_http_range_not_satisfiable(ngx_http_request_t *r); -static ngx_int_t ngx_http_range_test_overlapped(ngx_http_request_t *r, - ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); static ngx_int_t ngx_http_range_singlepart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); static ngx_int_t ngx_http_range_multipart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); +static ngx_int_t ngx_http_range_link_boundary_header(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t ***lll); +static ngx_int_t ngx_http_range_link_last_boundary(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t **ll); static ngx_int_t ngx_http_range_header_filter_init(ngx_conf_t *cf); static ngx_int_t ngx_http_range_body_filter_init(ngx_conf_t *cf); @@ -270,9 +276,9 @@ ngx_http_range_parse(ngx_http_request_t ngx_uint_t ranges) { u_char *p; - off_t start, end, size, content_length, cutoff, - cutlim; - ngx_uint_t suffix; + off_t start, end, size, content_length, + suffix_length, cutoff, cutlim; + ngx_uint_t i, suffix; ngx_http_range_t *range; ngx_http_range_filter_ctx_t *mctx; @@ -280,6 +286,7 @@ ngx_http_range_parse(ngx_http_request_t mctx = ngx_http_get_module_ctx(r->main, ngx_http_range_body_filter_module); if (mctx) { + ctx->boundary_header = mctx->boundary_header; ctx->ranges = mctx->ranges; return NGX_OK; } @@ -293,6 +300,8 @@ ngx_http_range_parse(ngx_http_request_t p = r->headers_in.range->value.data + 6; size = 0; + range = NULL; + suffix_length = 0; content_length = r->headers_out.content_length_n; cutoff = NGX_MAX_OFF_T_VALUE / 10; @@ -374,6 +383,94 @@ ngx_http_range_parse(ngx_http_request_t return NGX_DECLINED; } + if (suffix) { + + if (end - start > suffix_length) { + suffix_length = end - start; + } + + } else { + + if (ctx->ranges.nelts) { + + if (start < range->start) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "descending order ranges %O-%O, %O-%O", + range->start, range->end - 1, + start, end - 1); + + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + + if (end <= range->end) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "inclusive ranges %O-%O, %O-%O", + range->start, range->end - 1, + start, end - 1); + + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + + if (start < range->end + NGX_HTTP_RANGE_MULTIPART_GAP) { + size -= range->end - range->start; + start = range->start; + ctx->ranges.nelts--; + } + } + + range = ngx_array_push(&ctx->ranges); + if (range == NULL) { + return NGX_ERROR; + } + + range->start = start; + range->end = end; + + if (size > NGX_MAX_OFF_T_VALUE - (end - start)) { + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + + size += end - start; + } + + } else if (start == 0) { + return NGX_DECLINED; + } + + if (*p++ != ',') { + break; + } + } + + if (suffix_length) { + start = content_length - suffix_length; + end = content_length; + + i = 0; + range = ctx->ranges.elts; + + while (ctx->ranges.nelts) { + + i = ctx->ranges.nelts - 1; + if (start >= range[i].start) { + break; + } + + size -= range[i].end - range[i].start; + ctx->ranges.nelts--; + } + + if (ctx->ranges.nelts + && start < range[i].end + NGX_HTTP_RANGE_MULTIPART_GAP) + { + size -= range[i].end - range[i].start; + start = range[i].start; + ctx->ranges.nelts--; + i--; + } + + if (ctx->ranges.nelts == 0 || end > range[i].end) { + range = ngx_array_push(&ctx->ranges); if (range == NULL) { return NGX_ERROR; @@ -387,13 +484,6 @@ ngx_http_range_parse(ngx_http_request_t } size += end - start; - - } else if (start == 0) { - return NGX_DECLINED; - } - - if (*p++ != ',') { - break; } } @@ -470,6 +560,10 @@ ngx_http_range_multipart_header(ngx_http ngx_http_range_t *range; ngx_atomic_uint_t boundary; + if (r != r->main) { + return ngx_http_next_header_filter(r); + } + size = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof(CRLF "Content-Type: ") - 1 + r->headers_out.content_type.len @@ -571,10 +665,11 @@ ngx_http_range_multipart_header(ngx_http - range[i].content_range.data; len += ctx->boundary_header.len + range[i].content_range.len - + (range[i].end - range[i].start); + + (range[i].end - range[i].start); } r->headers_out.content_length_n = len; + r->headers_out.content_offset = range[0].start; if (r->headers_out.content_length) { r->headers_out.content_length->hash = 0; @@ -640,63 +735,15 @@ ngx_http_range_body_filter(ngx_http_requ return ngx_http_range_singlepart_body(r, ctx, in); } - /* - * multipart ranges are supported only if whole body is in a single buffer - */ - if (ngx_buf_special(in->buf)) { return ngx_http_next_body_filter(r, in); } - if (ngx_http_range_test_overlapped(r, ctx, in) != NGX_OK) { - return NGX_ERROR; - } - return ngx_http_range_multipart_body(r, ctx, in); } static ngx_int_t -ngx_http_range_test_overlapped(ngx_http_request_t *r, - ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) -{ - off_t start, last; - ngx_buf_t *buf; - ngx_uint_t i; - ngx_http_range_t *range; - - if (ctx->offset) { - goto overlapped; - } - - buf = in->buf; - - if (!buf->last_buf) { - start = ctx->offset; - last = ctx->offset + ngx_buf_size(buf); - - range = ctx->ranges.elts; - for (i = 0; i < ctx->ranges.nelts; i++) { - if (start > range[i].start || last < range[i].end) { - goto overlapped; - } - } - } - - ctx->offset = ngx_buf_size(buf); - - return NGX_OK; - -overlapped: - - ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, - "range in overlapped buffers"); - - return NGX_ERROR; -} - - -static ngx_int_t ngx_http_range_singlepart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) { @@ -787,96 +834,225 @@ static ngx_int_t ngx_http_range_multipart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) { - ngx_buf_t *b, *buf; - ngx_uint_t i; - ngx_chain_t *out, *hcl, *rcl, *dcl, **ll; - ngx_http_range_t *range; + off_t start, last, back; + ngx_buf_t *buf, *b; + ngx_uint_t i, finished; + ngx_chain_t *out, *cl, *ncl, **ll; + ngx_http_range_t *range, *tail; - ll = &out; - buf = in->buf; range = ctx->ranges.elts; - for (i = 0; i < ctx->ranges.nelts; i++) { + if (!ctx->index) { + for (i = 0; i < ctx->ranges.nelts; i++) { + if (ctx->offset < range[i].end) { + ctx->index = i + 1; + break; + } + } + } + + tail = range + ctx->ranges.nelts - 1; + range += ctx->index - 1; + + out = NULL; + ll = &out; + finished = 0; - /* - * The boundary header of the range: - * CRLF - * "--0123456789" CRLF - * "Content-Type: image/jpeg" CRLF - * "Content-Range: bytes " - */ + for (cl = in; cl; cl = cl->next) { + + buf = cl->buf; + + start = ctx->offset; + last = ctx->offset + ngx_buf_size(buf); - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + ctx->offset = last; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body buf: %O-%O", start, last); + + if (ngx_buf_special(buf)) { + *ll = cl; + ll = &cl->next; + continue; } - b->memory = 1; - b->pos = ctx->boundary_header.data; - b->last = ctx->boundary_header.data + ctx->boundary_header.len; + if (range->end <= start || range->start >= last) { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body skip"); - hcl = ngx_alloc_chain_link(r->pool); - if (hcl == NULL) { - return NGX_ERROR; + if (buf->in_file) { + buf->file_pos = buf->file_last; + } + + buf->pos = buf->last; + buf->sync = 1; + + continue; } - hcl->buf = b; + if (range->start >= start) { + if (ngx_http_range_link_boundary_header(r, ctx, &ll) != NGX_OK) { + return NGX_ERROR; + } - /* "SSSS-EEEE/TTTT" CRLF CRLF */ + if (buf->in_file) { + buf->file_pos += range->start - start; + } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + if (ngx_buf_in_memory(buf)) { + buf->pos += (size_t) (range->start - start); + } } - b->temporary = 1; - b->pos = range[i].content_range.data; - b->last = range[i].content_range.data + range[i].content_range.len; + if (range->end <= last) { + + if (range < tail && range[1].start < last) { + + b = ngx_alloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } + + ncl = ngx_alloc_chain_link(r->pool); + if (ncl == NULL) { + return NGX_ERROR; + } - rcl = ngx_alloc_chain_link(r->pool); - if (rcl == NULL) { - return NGX_ERROR; - } + ncl->buf = b; + ncl->next = cl->next; + cl->next = ncl; + + ngx_memcpy(b, buf, sizeof(ngx_buf_t)); + buf->last_in_chain = 0; + buf->last_buf = 0; + + back = last - range->end; + ctx->offset -= back; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body reuse buf: %O-%O", + ctx->offset, ctx->offset + back); - rcl->buf = b; + if (b->in_file) { + b->file_pos = b->file_last - back; + } + if (ngx_buf_in_memory(b)) { + b->pos = b->last - back; + } + } + + if (buf->in_file) { + buf->file_last -= last - range->end; + } - /* the range data */ + if (ngx_buf_in_memory(buf)) { + buf->last -= (size_t) (last - range->end); + } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + if (range == tail) { + buf->last_buf = (r == r->main) ? 1 : 0; + buf->last_in_chain = 1; + *ll = cl; + ll = &cl->next; + + finished = 1; + break; + } + + range++; + ctx->index++; } - b->in_file = buf->in_file; - b->temporary = buf->temporary; - b->memory = buf->memory; - b->mmap = buf->mmap; - b->file = buf->file; + *ll = cl; + ll = &cl->next; + } + + if (out == NULL) { + return NGX_OK; + } + + *ll = NULL; + + if (finished + && ngx_http_range_link_last_boundary(r, ctx, ll) != NGX_OK) + { + return NGX_ERROR; + } + + return ngx_http_next_body_filter(r, out); +} + - if (buf->in_file) { - b->file_pos = buf->file_pos + range[i].start; - b->file_last = buf->file_pos + range[i].end; - } +static ngx_int_t +ngx_http_range_link_boundary_header(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t ***lll) +{ + ngx_buf_t *b; + ngx_chain_t *hcl, *rcl; + ngx_http_range_t *range; + + /* + * The boundary header of the range: + * CRLF + * "--0123456789" CRLF + * "Content-Type: image/jpeg" CRLF + * "Content-Range: bytes " + */ + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } - if (ngx_buf_in_memory(buf)) { - b->pos = buf->pos + (size_t) range[i].start; - b->last = buf->pos + (size_t) range[i].end; - } + b->memory = 1; + b->pos = ctx->boundary_header.data; + b->last = ctx->boundary_header.data + ctx->boundary_header.len; + + hcl = ngx_alloc_chain_link(r->pool); + if (hcl == NULL) { + return NGX_ERROR; + } + + hcl->buf = b; + + + /* "SSSS-EEEE/TTTT" CRLF CRLF */ + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } - dcl = ngx_alloc_chain_link(r->pool); - if (dcl == NULL) { - return NGX_ERROR; - } + range = ctx->ranges.elts; + b->temporary = 1; + b->pos = range[ctx->index - 1].content_range.data; + b->last = range[ctx->index - 1].content_range.data + + range[ctx->index - 1].content_range.len; + + rcl = ngx_alloc_chain_link(r->pool); + if (rcl == NULL) { + return NGX_ERROR; + } + + rcl->buf = b; - dcl->buf = b; + **lll = hcl; + hcl->next = rcl; + *lll = &rcl->next; + + return NGX_OK; +} - *ll = hcl; - hcl->next = rcl; - rcl->next = dcl; - ll = &dcl->next; - } + +static ngx_int_t +ngx_http_range_link_last_boundary(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t **ll) +{ + ngx_buf_t *b; + ngx_chain_t *hcl; /* the last boundary CRLF "--0123456789--" CRLF */ @@ -886,7 +1062,8 @@ ngx_http_range_multipart_body(ngx_http_r } b->temporary = 1; - b->last_buf = 1; + b->last_in_chain = 1; + b->last_buf = (r == r->main) ? 1 : 0; b->pos = ngx_pnalloc(r->pool, sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof("--" CRLF) - 1); @@ -909,7 +1086,7 @@ ngx_http_range_multipart_body(ngx_http_r *ll = hcl; - return ngx_http_next_body_filter(r, out); + return NGX_OK; } From hucong.c at foxmail.com Thu Oct 26 18:27:27 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 02:27:27 +0800 Subject: [patch] Range filter: move ranges array to ngx_http_headers_out_t. Message-ID: # HG changeset patch # User hucongcong # Date 1509041509 -28800 # Fri Oct 27 02:11:49 2017 +0800 # Node ID f7b01b22c785c8ee7976efed0c6c7cb71c6a511d # Parent 798418b3f9c21811debf6cbeaea3eb554c886905 Range filter: move ranges array to ngx_http_headers_out_t. diff -r 798418b3f9c2 -r f7b01b22c785 src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 01:21:46 2017 +0800 +++ b/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 02:11:49 2017 +0800 @@ -49,17 +49,10 @@ typedef struct { - off_t start; - off_t end; - ngx_str_t content_range; -} ngx_http_range_t; - + off_t offset; + ngx_uint_t index; /* start with 1 */ -typedef struct { - off_t offset; - ngx_str_t boundary_header; - ngx_uint_t index; /* start with 1 */ - ngx_array_t ranges; + ngx_str_t boundary_header; } ngx_http_range_filter_ctx_t; @@ -240,7 +233,7 @@ parse: r->headers_out.status = NGX_HTTP_PARTIAL_CONTENT; r->headers_out.status_line.len = 0; - if (ctx->ranges.nelts == 1) { + if (r->headers_out.ranges->nelts == 1) { return ngx_http_range_singlepart_header(r, ctx); } @@ -286,15 +279,15 @@ ngx_http_range_parse(ngx_http_request_t mctx = ngx_http_get_module_ctx(r->main, ngx_http_range_body_filter_module); if (mctx) { + r->headers_out.ranges = r->main->headers_out.ranges; ctx->boundary_header = mctx->boundary_header; - ctx->ranges = mctx->ranges; return NGX_OK; } } - if (ngx_array_init(&ctx->ranges, r->pool, 1, sizeof(ngx_http_range_t)) - != NGX_OK) - { + r->headers_out.ranges = ngx_array_create(r->pool, 1, + sizeof(ngx_http_range_t)); + if (r->headers_out.ranges == NULL) { return NGX_ERROR; } @@ -391,7 +384,7 @@ ngx_http_range_parse(ngx_http_request_t } else { - if (ctx->ranges.nelts) { + if (r->headers_out.ranges->nelts) { if (start < range->start) { ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, @@ -414,11 +407,11 @@ ngx_http_range_parse(ngx_http_request_t if (start < range->end + NGX_HTTP_RANGE_MULTIPART_GAP) { size -= range->end - range->start; start = range->start; - ctx->ranges.nelts--; + r->headers_out.ranges->nelts--; } } - range = ngx_array_push(&ctx->ranges); + range = ngx_array_push(r->headers_out.ranges); if (range == NULL) { return NGX_ERROR; } @@ -447,31 +440,31 @@ ngx_http_range_parse(ngx_http_request_t end = content_length; i = 0; - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; - while (ctx->ranges.nelts) { + while (r->headers_out.ranges->nelts) { - i = ctx->ranges.nelts - 1; + i = r->headers_out.ranges->nelts - 1; if (start >= range[i].start) { break; } size -= range[i].end - range[i].start; - ctx->ranges.nelts--; + r->headers_out.ranges->nelts--; } - if (ctx->ranges.nelts + if (r->headers_out.ranges->nelts && start < range[i].end + NGX_HTTP_RANGE_MULTIPART_GAP) { size -= range[i].end - range[i].start; start = range[i].start; - ctx->ranges.nelts--; + r->headers_out.ranges->nelts--; i--; } - if (ctx->ranges.nelts == 0 || end > range[i].end) { + if (r->headers_out.ranges->nelts == 0 || end > range[i].end) { - range = ngx_array_push(&ctx->ranges); + range = ngx_array_push(r->headers_out.ranges); if (range == NULL) { return NGX_ERROR; } @@ -487,7 +480,7 @@ ngx_http_range_parse(ngx_http_request_t } } - if (ctx->ranges.nelts == 0) { + if (r->headers_out.ranges->nelts == 0) { return NGX_HTTP_RANGE_NOT_SATISFIABLE; } @@ -530,7 +523,7 @@ ngx_http_range_singlepart_header(ngx_htt /* "Content-Range: bytes SSSS-EEEE/TTTT" header */ - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; content_range->value.len = ngx_sprintf(content_range->value.data, "bytes %O-%O/%O", @@ -646,8 +639,8 @@ ngx_http_range_multipart_header(ngx_http len = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof("--" CRLF) - 1; - range = ctx->ranges.elts; - for (i = 0; i < ctx->ranges.nelts; i++) { + range = r->headers_out.ranges->elts; + for (i = 0; i < r->headers_out.ranges->nelts; i++) { /* the size of the range: "SSSS-EEEE/TTTT" CRLF CRLF */ @@ -731,7 +724,7 @@ ngx_http_range_body_filter(ngx_http_requ return ngx_http_next_body_filter(r, in); } - if (ctx->ranges.nelts == 1) { + if (r->headers_out.ranges->nelts == 1) { return ngx_http_range_singlepart_body(r, ctx, in); } @@ -754,7 +747,7 @@ ngx_http_range_singlepart_body(ngx_http_ out = NULL; ll = &out; - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; for (cl = in; cl; cl = cl->next) { @@ -840,10 +833,10 @@ ngx_http_range_multipart_body(ngx_http_r ngx_chain_t *out, *cl, *ncl, **ll; ngx_http_range_t *range, *tail; - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; if (!ctx->index) { - for (i = 0; i < ctx->ranges.nelts; i++) { + for (i = 0; i < r->headers_out.ranges->nelts; i++) { if (ctx->offset < range[i].end) { ctx->index = i + 1; break; @@ -851,7 +844,7 @@ ngx_http_range_multipart_body(ngx_http_r } } - tail = range + ctx->ranges.nelts - 1; + tail = range + r->headers_out.ranges->nelts - 1; range += ctx->index - 1; out = NULL; @@ -1026,7 +1019,7 @@ ngx_http_range_link_boundary_header(ngx_ return NGX_ERROR; } - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; b->temporary = 1; b->pos = range[ctx->index - 1].content_range.data; b->last = range[ctx->index - 1].content_range.data diff -r 798418b3f9c2 -r f7b01b22c785 src/http/ngx_http_request.h --- a/src/http/ngx_http_request.h Fri Oct 27 01:21:46 2017 +0800 +++ b/src/http/ngx_http_request.h Fri Oct 27 02:11:49 2017 +0800 @@ -251,6 +251,13 @@ typedef struct { typedef struct { + off_t start; + off_t end; + ngx_str_t content_range; +} ngx_http_range_t; + + +typedef struct { ngx_list_t headers; ngx_list_t trailers; @@ -278,6 +285,7 @@ typedef struct { u_char *content_type_lowcase; ngx_uint_t content_type_hash; + ngx_array_t *ranges; /* ngx_http_range_t */ ngx_array_t cache_control; off_t content_length_n; From hucong.c at foxmail.com Thu Oct 26 18:28:17 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 02:28:17 +0800 Subject: Slice filter: support multiple ranges. Message-ID: # HG changeset patch # User hucongcong # Date 1509041837 -28800 # Fri Oct 27 02:17:17 2017 +0800 # Node ID 2366fd2d2aa2820ab846fb51bba0e399fbb7ecc0 # Parent f7b01b22c785c8ee7976efed0c6c7cb71c6a511d Slice filter: support multiple ranges. diff -r f7b01b22c785 -r 2366fd2d2aa2 src/http/modules/ngx_http_slice_filter_module.c --- a/src/http/modules/ngx_http_slice_filter_module.c Fri Oct 27 02:11:49 2017 +0800 +++ b/src/http/modules/ngx_http_slice_filter_module.c Fri Oct 27 02:17:17 2017 +0800 @@ -22,6 +22,8 @@ typedef struct { ngx_str_t etag; unsigned last:1; unsigned active:1; + unsigned multipart:1; + ngx_uint_t index; ngx_http_request_t *sr; } ngx_http_slice_ctx_t; @@ -103,7 +105,9 @@ ngx_http_slice_header_filter(ngx_http_re { off_t end; ngx_int_t rc; + ngx_uint_t i; ngx_table_elt_t *h; + ngx_http_range_t *range; ngx_http_slice_ctx_t *ctx; ngx_http_slice_loc_conf_t *slcf; ngx_http_slice_content_range_t cr; @@ -182,27 +186,48 @@ ngx_http_slice_header_filter(ngx_http_re r->allow_ranges = 1; r->subrequest_ranges = 1; - r->single_range = 1; rc = ngx_http_next_header_filter(r); - if (r != r->main) { - return rc; + if (r == r->main) { + r->preserve_body = 1; + + if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { + ctx->multipart = (r->headers_out.ranges->nelts != 1); + range = r->headers_out.ranges->elts; + + if (ctx->start + (off_t) slcf->size <= range[0].start) { + ctx->start = slcf->size * (range[0].start / slcf->size); + } + + ctx->end = range[r->headers_out.ranges->nelts - 1].end; + + } else { + ctx->end = cr.complete_length; + } } - r->preserve_body = 1; + if (ctx->multipart) { + end = ctx->start + (off_t) slcf->size - 1; + + range = r->headers_out.ranges->elts; + for (i = ctx->index; i < r->headers_out.ranges->nelts - 1; i++) { + + if (end < range[i].end) { + break; + } - if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { - if (ctx->start + (off_t) slcf->size <= r->headers_out.content_offset) { - ctx->start = slcf->size - * (r->headers_out.content_offset / slcf->size); + if (end < range[i + 1].start && ctx->start >= range[i].end) { + i++; + ctx->index = i; + ctx->start = slcf->size * (range[i].start / slcf->size); + + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "range multipart so fast forward to %O-%O @%O", + range[i].start, range[i].end, ctx->start); + break; + } } - - ctx->end = r->headers_out.content_offset - + r->headers_out.content_length_n; - - } else { - ctx->end = cr.complete_length; } return rc; From dnj0496 at gmail.com Thu Oct 26 21:04:44 2017 From: dnj0496 at gmail.com (Dk Jack) Date: Thu, 26 Oct 2017 14:04:44 -0700 Subject: monitor file Message-ID: Hi, In my module I'd like to monitor the creation of a file (at a specific) using inotify or something similar and load some data from the file. How would I go about hooking up the inotifyFd with the nginx event system so that I get a callback when the file is created on the disk? Dk. -------------- next part -------------- An HTML attachment was scrubbed... URL: From hucong.c at foxmail.com Fri Oct 27 04:12:51 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 12:12:51 +0800 Subject: monitor file In-Reply-To: References: Message-ID: Hi, On Friday, Oct 27, 2017 5:04 AM, Dk Jack wrote: >In my module I'd like to monitor the creation of a file (at a specific) using >inotify or something similar and load some data from the file. How would I go >about hooking up the inotifyFd with the nginx event system so that I get a >callback when the file is created on the disk? First of all, please make sure that this module is behind ngx_event_core_module. Here, we asume the module type is NGX_HTTP_MODULE. Secondly, get some configuration information. Thirdly, module.init_process needs to be implemented. And here are the outline of init_process: if (ngx_worker == 0) { fd = inotify_init1(flags); wd = inotify_add_watch(fd, pathname, mask); c = ngx_get_connection(fd, log); rev = c->read; ...; rev->handler = your_main_callback_handler; /* What you want to do */ ngx_add_event(rev, NGX_READ_EVENT, 0); } Hope this helps! From dnj0496 at gmail.com Fri Oct 27 04:25:01 2017 From: dnj0496 at gmail.com (Dk Jack) Date: Thu, 26 Oct 2017 21:25:01 -0700 Subject: monitor file In-Reply-To: References: Message-ID: Thanks for the quick response. This is exactly what I need. Yes, it is a NGX_HTTP_MODULE. How do I ensure, "this module is behind ngx_event_core_module"? Thanks. Dk On Thu, Oct 26, 2017 at 9:12 PM, ?? (hucc) wrote: > Hi, > > On Friday, Oct 27, 2017 5:04 AM, Dk Jack wrote: > > >In my module I'd like to monitor the creation of a file (at a specific) > using > >inotify or something similar and load some data from the file. How would > I go > >about hooking up the inotifyFd with the nginx event system so that I get a > >callback when the file is created on the disk? > > First of all, please make sure that this module is behind > ngx_event_core_module. > Here, we asume the module type is NGX_HTTP_MODULE. > > Secondly, get some configuration information. > > Thirdly, module.init_process needs to be implemented. > And here are the outline of init_process: > if (ngx_worker == 0) { > fd = inotify_init1(flags); > wd = inotify_add_watch(fd, pathname, mask); > c = ngx_get_connection(fd, log); > rev = c->read; > ...; > rev->handler = your_main_callback_handler; /* What you want to do */ > ngx_add_event(rev, NGX_READ_EVENT, 0); > } > > Hope this helps! > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > http://mailman.nginx.org/mailman/listinfo/nginx-devel > -------------- next part -------------- An HTML attachment was scrubbed... URL: From hucong.c at foxmail.com Fri Oct 27 05:19:37 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 13:19:37 +0800 Subject: monitor file In-Reply-To: References: Message-ID: hello, On Friday, Oct 27, 2017 12:25 PM, Dk Jack wrote: >On Thu, Oct 26, 2017 at 9:12 PM, ?? (hucc) wrote: > >>On Friday, Oct 27, 2017 5:04 AM, Dk Jack wrote: >> >>>In my module I'd like to monitor the creation of a file (at a specific) using >>>inotify or something similar and load some data from the file. How would I go >>>about hooking up the inotifyFd with the nginx event system so that I get a >>>callback when the file is created on the disk? >> >> >>First of all, please make sure that this module is behind ngx_event_core_module. >>Here, we asume the module type is NGX_HTTP_MODULE. >> >>Secondly, get some configuration information. >> >>Thirdly, module.init_process needs to be implemented. And here are >>the outline of init_process: >>if (ngx_worker == 0) { >> fd = inotify_init1(flags); >> wd = inotify_add_watch(fd, pathname, mask); >> c = ngx_get_connection(fd, log); >> rev = c->read; >> ...; >> rev->handler = your_main_callback_handler; /* What you want to do */ >> ngx_add_event(rev, NGX_READ_EVENT, 0); >>} >> >>Hope this helps! > >.... This is exactly what I need. Yes, it is a NGX_HTTP_MODULE. >How do I ensure, "this module is behind ngx_event_core_module"? Thanks. If the type is NGX_HTTP_MODULE, then the module is already behind ngx_event_core_module. From hucong.c at foxmail.com Fri Oct 27 10:44:32 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 18:44:32 +0800 Subject: [patch] Range filter: more appropriate restriction on max ranges. In-Reply-To: References: Message-ID: Hi, Please ignore this patch, I will update later. I am sorry? ------------------ Original ------------------ From: "?? (hucc)";; Date: Oct 27, 2017 To: "nginx-devel"; Subject: [patch] Range filter: more appropriate restriction on max ranges. # HG changeset patch # User hucongcong # Date 1509035936 -28800 # Fri Oct 27 00:38:56 2017 +0800 # Node ID a429a3b21a3950d69781e109111831e7c5f33566 # Parent 9ef704d8563af4aff6817ab1c694fb40591f20b3 Range filter: more appropriate restriction on max ranges. diff -r 9ef704d8563a -r a429a3b21a39 src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c Tue Oct 17 19:52:16 2017 +0300 +++ b/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 00:38:56 2017 +0800 @@ -369,6 +369,11 @@ ngx_http_range_parse(ngx_http_request_t found: if (start < end) { + + if (ranges-- == 0) { + return NGX_DECLINED; + } + range = ngx_array_push(&ctx->ranges); if (range == NULL) { return NGX_ERROR; @@ -383,10 +388,6 @@ ngx_http_range_parse(ngx_http_request_t size += end - start; - if (ranges-- == 0) { - return NGX_DECLINED; - } - } else if (start == 0) { return NGX_DECLINED; } _______________________________________________ nginx-devel mailing list nginx-devel at nginx.org http://mailman.nginx.org/mailman/listinfo/nginx-devel From hucong.c at foxmail.com Fri Oct 27 10:45:32 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 18:45:32 +0800 Subject: [patch] Range filter: support multiple ranges. In-Reply-To: References: Message-ID: Hi, Please ignore this patch, I will update later. I am sorry? ------------------ Original ------------------ From: "?? (hucc)";; Date: Oct 27, 2017 To: "nginx-devel"; Subject: [patch] Range filter: support multiple ranges. # HG changeset patch # User hucongcong # Date 1509038506 -28800 # Fri Oct 27 01:21:46 2017 +0800 # Node ID 798418b3f9c21811debf6cbeaea3eb554c886905 # Parent a429a3b21a3950d69781e109111831e7c5f33566 Range filter: support multiple ranges. Nginx will return 416 when the ranges (first-byte-pos) are not listed in ascending order or when one byte-range-spec contains another. And besides, nginx will coalesce any of the ranges that overlap, or that are separated by a gap that is smaller than the NGX_HTTP_RANGE_MULTIPART_GAP macro. diff -r a429a3b21a39 -r 798418b3f9c2 src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 00:38:56 2017 +0800 +++ b/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 01:21:46 2017 +0800 @@ -10,6 +10,9 @@ #include +#define NGX_HTTP_RANGE_MULTIPART_GAP 80 + + /* * the single part format: * @@ -55,6 +58,7 @@ typedef struct { typedef struct { off_t offset; ngx_str_t boundary_header; + ngx_uint_t index; /* start with 1 */ ngx_array_t ranges; } ngx_http_range_filter_ctx_t; @@ -66,12 +70,14 @@ static ngx_int_t ngx_http_range_singlepa static ngx_int_t ngx_http_range_multipart_header(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx); static ngx_int_t ngx_http_range_not_satisfiable(ngx_http_request_t *r); -static ngx_int_t ngx_http_range_test_overlapped(ngx_http_request_t *r, - ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); static ngx_int_t ngx_http_range_singlepart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); static ngx_int_t ngx_http_range_multipart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); +static ngx_int_t ngx_http_range_link_boundary_header(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t ***lll); +static ngx_int_t ngx_http_range_link_last_boundary(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t **ll); static ngx_int_t ngx_http_range_header_filter_init(ngx_conf_t *cf); static ngx_int_t ngx_http_range_body_filter_init(ngx_conf_t *cf); @@ -270,9 +276,9 @@ ngx_http_range_parse(ngx_http_request_t ngx_uint_t ranges) { u_char *p; - off_t start, end, size, content_length, cutoff, - cutlim; - ngx_uint_t suffix; + off_t start, end, size, content_length, + suffix_length, cutoff, cutlim; + ngx_uint_t i, suffix; ngx_http_range_t *range; ngx_http_range_filter_ctx_t *mctx; @@ -280,6 +286,7 @@ ngx_http_range_parse(ngx_http_request_t mctx = ngx_http_get_module_ctx(r->main, ngx_http_range_body_filter_module); if (mctx) { + ctx->boundary_header = mctx->boundary_header; ctx->ranges = mctx->ranges; return NGX_OK; } @@ -293,6 +300,8 @@ ngx_http_range_parse(ngx_http_request_t p = r->headers_in.range->value.data + 6; size = 0; + range = NULL; + suffix_length = 0; content_length = r->headers_out.content_length_n; cutoff = NGX_MAX_OFF_T_VALUE / 10; @@ -374,6 +383,94 @@ ngx_http_range_parse(ngx_http_request_t return NGX_DECLINED; } + if (suffix) { + + if (end - start > suffix_length) { + suffix_length = end - start; + } + + } else { + + if (ctx->ranges.nelts) { + + if (start < range->start) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "descending order ranges %O-%O, %O-%O", + range->start, range->end - 1, + start, end - 1); + + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + + if (end <= range->end) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "inclusive ranges %O-%O, %O-%O", + range->start, range->end - 1, + start, end - 1); + + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + + if (start < range->end + NGX_HTTP_RANGE_MULTIPART_GAP) { + size -= range->end - range->start; + start = range->start; + ctx->ranges.nelts--; + } + } + + range = ngx_array_push(&ctx->ranges); + if (range == NULL) { + return NGX_ERROR; + } + + range->start = start; + range->end = end; + + if (size > NGX_MAX_OFF_T_VALUE - (end - start)) { + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + + size += end - start; + } + + } else if (start == 0) { + return NGX_DECLINED; + } + + if (*p++ != ',') { + break; + } + } + + if (suffix_length) { + start = content_length - suffix_length; + end = content_length; + + i = 0; + range = ctx->ranges.elts; + + while (ctx->ranges.nelts) { + + i = ctx->ranges.nelts - 1; + if (start >= range[i].start) { + break; + } + + size -= range[i].end - range[i].start; + ctx->ranges.nelts--; + } + + if (ctx->ranges.nelts + && start < range[i].end + NGX_HTTP_RANGE_MULTIPART_GAP) + { + size -= range[i].end - range[i].start; + start = range[i].start; + ctx->ranges.nelts--; + i--; + } + + if (ctx->ranges.nelts == 0 || end > range[i].end) { + range = ngx_array_push(&ctx->ranges); if (range == NULL) { return NGX_ERROR; @@ -387,13 +484,6 @@ ngx_http_range_parse(ngx_http_request_t } size += end - start; - - } else if (start == 0) { - return NGX_DECLINED; - } - - if (*p++ != ',') { - break; } } @@ -470,6 +560,10 @@ ngx_http_range_multipart_header(ngx_http ngx_http_range_t *range; ngx_atomic_uint_t boundary; + if (r != r->main) { + return ngx_http_next_header_filter(r); + } + size = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof(CRLF "Content-Type: ") - 1 + r->headers_out.content_type.len @@ -571,10 +665,11 @@ ngx_http_range_multipart_header(ngx_http - range[i].content_range.data; len += ctx->boundary_header.len + range[i].content_range.len - + (range[i].end - range[i].start); + + (range[i].end - range[i].start); } r->headers_out.content_length_n = len; + r->headers_out.content_offset = range[0].start; if (r->headers_out.content_length) { r->headers_out.content_length->hash = 0; @@ -640,63 +735,15 @@ ngx_http_range_body_filter(ngx_http_requ return ngx_http_range_singlepart_body(r, ctx, in); } - /* - * multipart ranges are supported only if whole body is in a single buffer - */ - if (ngx_buf_special(in->buf)) { return ngx_http_next_body_filter(r, in); } - if (ngx_http_range_test_overlapped(r, ctx, in) != NGX_OK) { - return NGX_ERROR; - } - return ngx_http_range_multipart_body(r, ctx, in); } static ngx_int_t -ngx_http_range_test_overlapped(ngx_http_request_t *r, - ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) -{ - off_t start, last; - ngx_buf_t *buf; - ngx_uint_t i; - ngx_http_range_t *range; - - if (ctx->offset) { - goto overlapped; - } - - buf = in->buf; - - if (!buf->last_buf) { - start = ctx->offset; - last = ctx->offset + ngx_buf_size(buf); - - range = ctx->ranges.elts; - for (i = 0; i < ctx->ranges.nelts; i++) { - if (start > range[i].start || last < range[i].end) { - goto overlapped; - } - } - } - - ctx->offset = ngx_buf_size(buf); - - return NGX_OK; - -overlapped: - - ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, - "range in overlapped buffers"); - - return NGX_ERROR; -} - - -static ngx_int_t ngx_http_range_singlepart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) { @@ -787,96 +834,225 @@ static ngx_int_t ngx_http_range_multipart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) { - ngx_buf_t *b, *buf; - ngx_uint_t i; - ngx_chain_t *out, *hcl, *rcl, *dcl, **ll; - ngx_http_range_t *range; + off_t start, last, back; + ngx_buf_t *buf, *b; + ngx_uint_t i, finished; + ngx_chain_t *out, *cl, *ncl, **ll; + ngx_http_range_t *range, *tail; - ll = &out; - buf = in->buf; range = ctx->ranges.elts; - for (i = 0; i < ctx->ranges.nelts; i++) { + if (!ctx->index) { + for (i = 0; i < ctx->ranges.nelts; i++) { + if (ctx->offset < range[i].end) { + ctx->index = i + 1; + break; + } + } + } + + tail = range + ctx->ranges.nelts - 1; + range += ctx->index - 1; + + out = NULL; + ll = &out; + finished = 0; - /* - * The boundary header of the range: - * CRLF - * "--0123456789" CRLF - * "Content-Type: image/jpeg" CRLF - * "Content-Range: bytes " - */ + for (cl = in; cl; cl = cl->next) { + + buf = cl->buf; + + start = ctx->offset; + last = ctx->offset + ngx_buf_size(buf); - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + ctx->offset = last; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body buf: %O-%O", start, last); + + if (ngx_buf_special(buf)) { + *ll = cl; + ll = &cl->next; + continue; } - b->memory = 1; - b->pos = ctx->boundary_header.data; - b->last = ctx->boundary_header.data + ctx->boundary_header.len; + if (range->end <= start || range->start >= last) { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body skip"); - hcl = ngx_alloc_chain_link(r->pool); - if (hcl == NULL) { - return NGX_ERROR; + if (buf->in_file) { + buf->file_pos = buf->file_last; + } + + buf->pos = buf->last; + buf->sync = 1; + + continue; } - hcl->buf = b; + if (range->start >= start) { + if (ngx_http_range_link_boundary_header(r, ctx, &ll) != NGX_OK) { + return NGX_ERROR; + } - /* "SSSS-EEEE/TTTT" CRLF CRLF */ + if (buf->in_file) { + buf->file_pos += range->start - start; + } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + if (ngx_buf_in_memory(buf)) { + buf->pos += (size_t) (range->start - start); + } } - b->temporary = 1; - b->pos = range[i].content_range.data; - b->last = range[i].content_range.data + range[i].content_range.len; + if (range->end <= last) { + + if (range < tail && range[1].start < last) { + + b = ngx_alloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } + + ncl = ngx_alloc_chain_link(r->pool); + if (ncl == NULL) { + return NGX_ERROR; + } - rcl = ngx_alloc_chain_link(r->pool); - if (rcl == NULL) { - return NGX_ERROR; - } + ncl->buf = b; + ncl->next = cl->next; + cl->next = ncl; + + ngx_memcpy(b, buf, sizeof(ngx_buf_t)); + buf->last_in_chain = 0; + buf->last_buf = 0; + + back = last - range->end; + ctx->offset -= back; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body reuse buf: %O-%O", + ctx->offset, ctx->offset + back); - rcl->buf = b; + if (b->in_file) { + b->file_pos = b->file_last - back; + } + if (ngx_buf_in_memory(b)) { + b->pos = b->last - back; + } + } + + if (buf->in_file) { + buf->file_last -= last - range->end; + } - /* the range data */ + if (ngx_buf_in_memory(buf)) { + buf->last -= (size_t) (last - range->end); + } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + if (range == tail) { + buf->last_buf = (r == r->main) ? 1 : 0; + buf->last_in_chain = 1; + *ll = cl; + ll = &cl->next; + + finished = 1; + break; + } + + range++; + ctx->index++; } - b->in_file = buf->in_file; - b->temporary = buf->temporary; - b->memory = buf->memory; - b->mmap = buf->mmap; - b->file = buf->file; + *ll = cl; + ll = &cl->next; + } + + if (out == NULL) { + return NGX_OK; + } + + *ll = NULL; + + if (finished + && ngx_http_range_link_last_boundary(r, ctx, ll) != NGX_OK) + { + return NGX_ERROR; + } + + return ngx_http_next_body_filter(r, out); +} + - if (buf->in_file) { - b->file_pos = buf->file_pos + range[i].start; - b->file_last = buf->file_pos + range[i].end; - } +static ngx_int_t +ngx_http_range_link_boundary_header(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t ***lll) +{ + ngx_buf_t *b; + ngx_chain_t *hcl, *rcl; + ngx_http_range_t *range; + + /* + * The boundary header of the range: + * CRLF + * "--0123456789" CRLF + * "Content-Type: image/jpeg" CRLF + * "Content-Range: bytes " + */ + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } - if (ngx_buf_in_memory(buf)) { - b->pos = buf->pos + (size_t) range[i].start; - b->last = buf->pos + (size_t) range[i].end; - } + b->memory = 1; + b->pos = ctx->boundary_header.data; + b->last = ctx->boundary_header.data + ctx->boundary_header.len; + + hcl = ngx_alloc_chain_link(r->pool); + if (hcl == NULL) { + return NGX_ERROR; + } + + hcl->buf = b; + + + /* "SSSS-EEEE/TTTT" CRLF CRLF */ + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } - dcl = ngx_alloc_chain_link(r->pool); - if (dcl == NULL) { - return NGX_ERROR; - } + range = ctx->ranges.elts; + b->temporary = 1; + b->pos = range[ctx->index - 1].content_range.data; + b->last = range[ctx->index - 1].content_range.data + + range[ctx->index - 1].content_range.len; + + rcl = ngx_alloc_chain_link(r->pool); + if (rcl == NULL) { + return NGX_ERROR; + } + + rcl->buf = b; - dcl->buf = b; + **lll = hcl; + hcl->next = rcl; + *lll = &rcl->next; + + return NGX_OK; +} - *ll = hcl; - hcl->next = rcl; - rcl->next = dcl; - ll = &dcl->next; - } + +static ngx_int_t +ngx_http_range_link_last_boundary(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t **ll) +{ + ngx_buf_t *b; + ngx_chain_t *hcl; /* the last boundary CRLF "--0123456789--" CRLF */ @@ -886,7 +1062,8 @@ ngx_http_range_multipart_body(ngx_http_r } b->temporary = 1; - b->last_buf = 1; + b->last_in_chain = 1; + b->last_buf = (r == r->main) ? 1 : 0; b->pos = ngx_pnalloc(r->pool, sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof("--" CRLF) - 1); @@ -909,7 +1086,7 @@ ngx_http_range_multipart_body(ngx_http_r *ll = hcl; - return ngx_http_next_body_filter(r, out); + return NGX_OK; } _______________________________________________ nginx-devel mailing list nginx-devel at nginx.org http://mailman.nginx.org/mailman/listinfo/nginx-devel From hucong.c at foxmail.com Fri Oct 27 10:46:18 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 18:46:18 +0800 Subject: [patch] Range filter: move ranges array to ngx_http_headers_out_t. In-Reply-To: References: Message-ID: Hi, Please ignore this patch, I will update later. I am sorry? ------------------ Original ------------------ From: "?? (hucc)";; Date: Oct 27, 2017 To: "nginx-devel"; Subject: [patch] Range filter: move ranges array to ngx_http_headers_out_t. # HG changeset patch # User hucongcong # Date 1509041509 -28800 # Fri Oct 27 02:11:49 2017 +0800 # Node ID f7b01b22c785c8ee7976efed0c6c7cb71c6a511d # Parent 798418b3f9c21811debf6cbeaea3eb554c886905 Range filter: move ranges array to ngx_http_headers_out_t. diff -r 798418b3f9c2 -r f7b01b22c785 src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 01:21:46 2017 +0800 +++ b/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 02:11:49 2017 +0800 @@ -49,17 +49,10 @@ typedef struct { - off_t start; - off_t end; - ngx_str_t content_range; -} ngx_http_range_t; - + off_t offset; + ngx_uint_t index; /* start with 1 */ -typedef struct { - off_t offset; - ngx_str_t boundary_header; - ngx_uint_t index; /* start with 1 */ - ngx_array_t ranges; + ngx_str_t boundary_header; } ngx_http_range_filter_ctx_t; @@ -240,7 +233,7 @@ parse: r->headers_out.status = NGX_HTTP_PARTIAL_CONTENT; r->headers_out.status_line.len = 0; - if (ctx->ranges.nelts == 1) { + if (r->headers_out.ranges->nelts == 1) { return ngx_http_range_singlepart_header(r, ctx); } @@ -286,15 +279,15 @@ ngx_http_range_parse(ngx_http_request_t mctx = ngx_http_get_module_ctx(r->main, ngx_http_range_body_filter_module); if (mctx) { + r->headers_out.ranges = r->main->headers_out.ranges; ctx->boundary_header = mctx->boundary_header; - ctx->ranges = mctx->ranges; return NGX_OK; } } - if (ngx_array_init(&ctx->ranges, r->pool, 1, sizeof(ngx_http_range_t)) - != NGX_OK) - { + r->headers_out.ranges = ngx_array_create(r->pool, 1, + sizeof(ngx_http_range_t)); + if (r->headers_out.ranges == NULL) { return NGX_ERROR; } @@ -391,7 +384,7 @@ ngx_http_range_parse(ngx_http_request_t } else { - if (ctx->ranges.nelts) { + if (r->headers_out.ranges->nelts) { if (start < range->start) { ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, @@ -414,11 +407,11 @@ ngx_http_range_parse(ngx_http_request_t if (start < range->end + NGX_HTTP_RANGE_MULTIPART_GAP) { size -= range->end - range->start; start = range->start; - ctx->ranges.nelts--; + r->headers_out.ranges->nelts--; } } - range = ngx_array_push(&ctx->ranges); + range = ngx_array_push(r->headers_out.ranges); if (range == NULL) { return NGX_ERROR; } @@ -447,31 +440,31 @@ ngx_http_range_parse(ngx_http_request_t end = content_length; i = 0; - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; - while (ctx->ranges.nelts) { + while (r->headers_out.ranges->nelts) { - i = ctx->ranges.nelts - 1; + i = r->headers_out.ranges->nelts - 1; if (start >= range[i].start) { break; } size -= range[i].end - range[i].start; - ctx->ranges.nelts--; + r->headers_out.ranges->nelts--; } - if (ctx->ranges.nelts + if (r->headers_out.ranges->nelts && start < range[i].end + NGX_HTTP_RANGE_MULTIPART_GAP) { size -= range[i].end - range[i].start; start = range[i].start; - ctx->ranges.nelts--; + r->headers_out.ranges->nelts--; i--; } - if (ctx->ranges.nelts == 0 || end > range[i].end) { + if (r->headers_out.ranges->nelts == 0 || end > range[i].end) { - range = ngx_array_push(&ctx->ranges); + range = ngx_array_push(r->headers_out.ranges); if (range == NULL) { return NGX_ERROR; } @@ -487,7 +480,7 @@ ngx_http_range_parse(ngx_http_request_t } } - if (ctx->ranges.nelts == 0) { + if (r->headers_out.ranges->nelts == 0) { return NGX_HTTP_RANGE_NOT_SATISFIABLE; } @@ -530,7 +523,7 @@ ngx_http_range_singlepart_header(ngx_htt /* "Content-Range: bytes SSSS-EEEE/TTTT" header */ - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; content_range->value.len = ngx_sprintf(content_range->value.data, "bytes %O-%O/%O", @@ -646,8 +639,8 @@ ngx_http_range_multipart_header(ngx_http len = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof("--" CRLF) - 1; - range = ctx->ranges.elts; - for (i = 0; i < ctx->ranges.nelts; i++) { + range = r->headers_out.ranges->elts; + for (i = 0; i < r->headers_out.ranges->nelts; i++) { /* the size of the range: "SSSS-EEEE/TTTT" CRLF CRLF */ @@ -731,7 +724,7 @@ ngx_http_range_body_filter(ngx_http_requ return ngx_http_next_body_filter(r, in); } - if (ctx->ranges.nelts == 1) { + if (r->headers_out.ranges->nelts == 1) { return ngx_http_range_singlepart_body(r, ctx, in); } @@ -754,7 +747,7 @@ ngx_http_range_singlepart_body(ngx_http_ out = NULL; ll = &out; - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; for (cl = in; cl; cl = cl->next) { @@ -840,10 +833,10 @@ ngx_http_range_multipart_body(ngx_http_r ngx_chain_t *out, *cl, *ncl, **ll; ngx_http_range_t *range, *tail; - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; if (!ctx->index) { - for (i = 0; i < ctx->ranges.nelts; i++) { + for (i = 0; i < r->headers_out.ranges->nelts; i++) { if (ctx->offset < range[i].end) { ctx->index = i + 1; break; @@ -851,7 +844,7 @@ ngx_http_range_multipart_body(ngx_http_r } } - tail = range + ctx->ranges.nelts - 1; + tail = range + r->headers_out.ranges->nelts - 1; range += ctx->index - 1; out = NULL; @@ -1026,7 +1019,7 @@ ngx_http_range_link_boundary_header(ngx_ return NGX_ERROR; } - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; b->temporary = 1; b->pos = range[ctx->index - 1].content_range.data; b->last = range[ctx->index - 1].content_range.data diff -r 798418b3f9c2 -r f7b01b22c785 src/http/ngx_http_request.h --- a/src/http/ngx_http_request.h Fri Oct 27 01:21:46 2017 +0800 +++ b/src/http/ngx_http_request.h Fri Oct 27 02:11:49 2017 +0800 @@ -251,6 +251,13 @@ typedef struct { typedef struct { + off_t start; + off_t end; + ngx_str_t content_range; +} ngx_http_range_t; + + +typedef struct { ngx_list_t headers; ngx_list_t trailers; @@ -278,6 +285,7 @@ typedef struct { u_char *content_type_lowcase; ngx_uint_t content_type_hash; + ngx_array_t *ranges; /* ngx_http_range_t */ ngx_array_t cache_control; off_t content_length_n; _______________________________________________ nginx-devel mailing list nginx-devel at nginx.org http://mailman.nginx.org/mailman/listinfo/nginx-devel From hucong.c at foxmail.com Fri Oct 27 10:47:01 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 18:47:01 +0800 Subject: Slice filter: support multiple ranges. In-Reply-To: References: Message-ID: Hi, Please ignore this patch, I will update later. I am sorry? ------------------ Original ------------------ From: "?? (hucc)";; Date: Oct 27, 2017 To: "nginx-devel"; Subject: Slice filter: support multiple ranges. # HG changeset patch # User hucongcong # Date 1509041837 -28800 # Fri Oct 27 02:17:17 2017 +0800 # Node ID 2366fd2d2aa2820ab846fb51bba0e399fbb7ecc0 # Parent f7b01b22c785c8ee7976efed0c6c7cb71c6a511d Slice filter: support multiple ranges. diff -r f7b01b22c785 -r 2366fd2d2aa2 src/http/modules/ngx_http_slice_filter_module.c --- a/src/http/modules/ngx_http_slice_filter_module.c Fri Oct 27 02:11:49 2017 +0800 +++ b/src/http/modules/ngx_http_slice_filter_module.c Fri Oct 27 02:17:17 2017 +0800 @@ -22,6 +22,8 @@ typedef struct { ngx_str_t etag; unsigned last:1; unsigned active:1; + unsigned multipart:1; + ngx_uint_t index; ngx_http_request_t *sr; } ngx_http_slice_ctx_t; @@ -103,7 +105,9 @@ ngx_http_slice_header_filter(ngx_http_re { off_t end; ngx_int_t rc; + ngx_uint_t i; ngx_table_elt_t *h; + ngx_http_range_t *range; ngx_http_slice_ctx_t *ctx; ngx_http_slice_loc_conf_t *slcf; ngx_http_slice_content_range_t cr; @@ -182,27 +186,48 @@ ngx_http_slice_header_filter(ngx_http_re r->allow_ranges = 1; r->subrequest_ranges = 1; - r->single_range = 1; rc = ngx_http_next_header_filter(r); - if (r != r->main) { - return rc; + if (r == r->main) { + r->preserve_body = 1; + + if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { + ctx->multipart = (r->headers_out.ranges->nelts != 1); + range = r->headers_out.ranges->elts; + + if (ctx->start + (off_t) slcf->size <= range[0].start) { + ctx->start = slcf->size * (range[0].start / slcf->size); + } + + ctx->end = range[r->headers_out.ranges->nelts - 1].end; + + } else { + ctx->end = cr.complete_length; + } } - r->preserve_body = 1; + if (ctx->multipart) { + end = ctx->start + (off_t) slcf->size - 1; + + range = r->headers_out.ranges->elts; + for (i = ctx->index; i < r->headers_out.ranges->nelts - 1; i++) { + + if (end < range[i].end) { + break; + } - if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { - if (ctx->start + (off_t) slcf->size <= r->headers_out.content_offset) { - ctx->start = slcf->size - * (r->headers_out.content_offset / slcf->size); + if (end < range[i + 1].start && ctx->start >= range[i].end) { + i++; + ctx->index = i; + ctx->start = slcf->size * (range[i].start / slcf->size); + + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "range multipart so fast forward to %O-%O @%O", + range[i].start, range[i].end, ctx->start); + break; + } } - - ctx->end = r->headers_out.content_offset - + r->headers_out.content_length_n; - - } else { - ctx->end = cr.complete_length; } return rc; _______________________________________________ nginx-devel mailing list nginx-devel at nginx.org http://mailman.nginx.org/mailman/listinfo/nginx-devel From hucong.c at foxmail.com Fri Oct 27 10:48:52 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 18:48:52 +0800 Subject: [bugfix] Range filter: more appropriate restriction on max ranges. Message-ID: # HG changeset patch # User hucongcong # Date 1509099660 -28800 # Fri Oct 27 18:21:00 2017 +0800 # Node ID b9850d3deb277bd433a689712c40a84401443520 # Parent 9ef704d8563af4aff6817ab1c694fb40591f20b3 Range filter: more appropriate restriction on max ranges. diff -r 9ef704d8563a -r b9850d3deb27 src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c Tue Oct 17 19:52:16 2017 +0300 +++ b/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 18:21:00 2017 +0800 @@ -369,6 +369,11 @@ ngx_http_range_parse(ngx_http_request_t found: if (start < end) { + + if (ranges-- == 0) { + return NGX_DECLINED; + } + range = ngx_array_push(&ctx->ranges); if (range == NULL) { return NGX_ERROR; @@ -383,10 +388,6 @@ ngx_http_range_parse(ngx_http_request_t size += end - start; - if (ranges-- == 0) { - return NGX_DECLINED; - } - } else if (start == 0) { return NGX_DECLINED; } From hucong.c at foxmail.com Fri Oct 27 10:50:32 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 18:50:32 +0800 Subject: [patch-1] Range filter: support multiple ranges. Message-ID: # HG changeset patch # User hucongcong # Date 1509099940 -28800 # Fri Oct 27 18:25:40 2017 +0800 # Node ID 62c100a0d42614cd46f0719c0acb0ad914594217 # Parent b9850d3deb277bd433a689712c40a84401443520 Range filter: support multiple ranges. When multiple ranges are requested, nginx will coalesce any of the ranges that overlap, or that are separated by a gap that is smaller than the NGX_HTTP_RANGE_MULTIPART_GAP macro. diff -r b9850d3deb27 -r 62c100a0d426 src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 18:21:00 2017 +0800 +++ b/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 18:25:40 2017 +0800 @@ -45,33 +45,34 @@ */ -typedef struct { - off_t start; - off_t end; - ngx_str_t content_range; -} ngx_http_range_t; +#define NGX_HTTP_RANGE_MULTIPART_GAP 80 typedef struct { - off_t offset; - ngx_str_t boundary_header; - ngx_array_t ranges; + off_t offset; + ngx_uint_t index; /* start with 1 */ + + ngx_str_t boundary_header; } ngx_http_range_filter_ctx_t; static ngx_int_t ngx_http_range_parse(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_uint_t ranges); +static int ngx_libc_cdecl ngx_http_range_cmp(const void *one, const void *two); +static void ngx_http_range_coalesce(ngx_http_request_t *r); static ngx_int_t ngx_http_range_singlepart_header(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx); static ngx_int_t ngx_http_range_multipart_header(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx); static ngx_int_t ngx_http_range_not_satisfiable(ngx_http_request_t *r); -static ngx_int_t ngx_http_range_test_overlapped(ngx_http_request_t *r, - ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); static ngx_int_t ngx_http_range_singlepart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); static ngx_int_t ngx_http_range_multipart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); +static ngx_int_t ngx_http_range_link_boundary_header(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t ***lll); +static ngx_int_t ngx_http_range_link_last_boundary(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t **ll); static ngx_int_t ngx_http_range_header_filter_init(ngx_conf_t *cf); static ngx_int_t ngx_http_range_body_filter_init(ngx_conf_t *cf); @@ -234,7 +235,7 @@ parse: r->headers_out.status = NGX_HTTP_PARTIAL_CONTENT; r->headers_out.status_line.len = 0; - if (ctx->ranges.nelts == 1) { + if (r->headers_out.ranges->nelts == 1) { return ngx_http_range_singlepart_header(r, ctx); } @@ -270,9 +271,9 @@ ngx_http_range_parse(ngx_http_request_t ngx_uint_t ranges) { u_char *p; - off_t start, end, size, content_length, cutoff, - cutlim; - ngx_uint_t suffix; + off_t start, end, content_length, + cutoff, cutlim; + ngx_uint_t suffix, unordered; ngx_http_range_t *range; ngx_http_range_filter_ctx_t *mctx; @@ -280,19 +281,21 @@ ngx_http_range_parse(ngx_http_request_t mctx = ngx_http_get_module_ctx(r->main, ngx_http_range_body_filter_module); if (mctx) { - ctx->ranges = mctx->ranges; + r->headers_out.ranges = r->main->headers_out.ranges; + ctx->boundary_header = mctx->boundary_header; return NGX_OK; } } - if (ngx_array_init(&ctx->ranges, r->pool, 1, sizeof(ngx_http_range_t)) - != NGX_OK) - { + r->headers_out.ranges = ngx_array_create(r->pool, 1, + sizeof(ngx_http_range_t)); + if (r->headers_out.ranges == NULL) { return NGX_ERROR; } p = r->headers_in.range->value.data + 6; - size = 0; + range = NULL; + unordered = 0; content_length = r->headers_out.content_length_n; cutoff = NGX_MAX_OFF_T_VALUE / 10; @@ -374,7 +377,22 @@ ngx_http_range_parse(ngx_http_request_t return NGX_DECLINED; } - range = ngx_array_push(&ctx->ranges); + if (range) { + + if (start < range->start) { + unordered++; + + } else if (start - NGX_HTTP_RANGE_MULTIPART_GAP < range->end) { + r->headers_out.ranges->nelts--; + start = range->start; + + if (range->end > end) { + end = range->end; + } + } + } + + range = ngx_array_push(r->headers_out.ranges); if (range == NULL) { return NGX_ERROR; } @@ -382,12 +400,6 @@ ngx_http_range_parse(ngx_http_request_t range->start = start; range->end = end; - if (size > NGX_MAX_OFF_T_VALUE - (end - start)) { - return NGX_HTTP_RANGE_NOT_SATISFIABLE; - } - - size += end - start; - } else if (start == 0) { return NGX_DECLINED; } @@ -397,15 +409,73 @@ ngx_http_range_parse(ngx_http_request_t } } - if (ctx->ranges.nelts == 0) { + if (unordered) { + ngx_qsort(r->headers_out.ranges->elts, r->headers_out.ranges->nelts, + sizeof(ngx_http_range_t), ngx_http_range_cmp); + + ngx_http_range_coalesce(r); + } + + if (r->headers_out.ranges->nelts == 0) { return NGX_HTTP_RANGE_NOT_SATISFIABLE; } - if (size > content_length) { - return NGX_DECLINED; + return NGX_OK; +} + + +static int ngx_libc_cdecl +ngx_http_range_cmp(const void *one, const void *two) +{ + ngx_http_range_t *first = (ngx_http_range_t *) one; + ngx_http_range_t *second = (ngx_http_range_t *) two; + + if (first->start > second->start) { + return 1; + } + + if (first->start < second->start) { + return -1; + } + + if (first->end < second->end) { + return 1; + } + + if (first->end > second->end) { + return -1; } - return NGX_OK; + return 0; +} + + +static void +ngx_http_range_coalesce(ngx_http_request_t *r) +{ + ngx_uint_t i, j; + ngx_http_range_t *range; + + range = r->headers_out.ranges->elts; + for (i = 0, j = 1; j < r->headers_out.ranges->nelts; j++) { + + if (range[j].end <= range[i].end) { + continue; + } + + if (range[j].start - NGX_HTTP_RANGE_MULTIPART_GAP < range[i].end) { + range[i].end = range[j].end; + continue; + } + + i++; + if (i != j) { + range[i].start = range[j].start; + range[i].end = range[j].end; + } + } + + r->headers_out.ranges->nelts = i + 1; } @@ -440,7 +510,7 @@ ngx_http_range_singlepart_header(ngx_htt /* "Content-Range: bytes SSSS-EEEE/TTTT" header */ - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; content_range->value.len = ngx_sprintf(content_range->value.data, "bytes %O-%O/%O", @@ -470,6 +540,10 @@ ngx_http_range_multipart_header(ngx_http ngx_http_range_t *range; ngx_atomic_uint_t boundary; + if (r != r->main) { + return ngx_http_next_header_filter(r); + } + size = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof(CRLF "Content-Type: ") - 1 + r->headers_out.content_type.len @@ -552,8 +626,8 @@ ngx_http_range_multipart_header(ngx_http len = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof("--" CRLF) - 1; - range = ctx->ranges.elts; - for (i = 0; i < ctx->ranges.nelts; i++) { + range = r->headers_out.ranges->elts; + for (i = 0; i < r->headers_out.ranges->nelts; i++) { /* the size of the range: "SSSS-EEEE/TTTT" CRLF CRLF */ @@ -571,10 +645,11 @@ ngx_http_range_multipart_header(ngx_http - range[i].content_range.data; len += ctx->boundary_header.len + range[i].content_range.len - + (range[i].end - range[i].start); + + (range[i].end - range[i].start); } r->headers_out.content_length_n = len; + r->headers_out.content_offset = range[0].start; if (r->headers_out.content_length) { r->headers_out.content_length->hash = 0; @@ -636,67 +711,19 @@ ngx_http_range_body_filter(ngx_http_requ return ngx_http_next_body_filter(r, in); } - if (ctx->ranges.nelts == 1) { + if (r->headers_out.ranges->nelts == 1) { return ngx_http_range_singlepart_body(r, ctx, in); } - /* - * multipart ranges are supported only if whole body is in a single buffer - */ - if (ngx_buf_special(in->buf)) { return ngx_http_next_body_filter(r, in); } - if (ngx_http_range_test_overlapped(r, ctx, in) != NGX_OK) { - return NGX_ERROR; - } - return ngx_http_range_multipart_body(r, ctx, in); } static ngx_int_t -ngx_http_range_test_overlapped(ngx_http_request_t *r, - ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) -{ - off_t start, last; - ngx_buf_t *buf; - ngx_uint_t i; - ngx_http_range_t *range; - - if (ctx->offset) { - goto overlapped; - } - - buf = in->buf; - - if (!buf->last_buf) { - start = ctx->offset; - last = ctx->offset + ngx_buf_size(buf); - - range = ctx->ranges.elts; - for (i = 0; i < ctx->ranges.nelts; i++) { - if (start > range[i].start || last < range[i].end) { - goto overlapped; - } - } - } - - ctx->offset = ngx_buf_size(buf); - - return NGX_OK; - -overlapped: - - ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, - "range in overlapped buffers"); - - return NGX_ERROR; -} - - -static ngx_int_t ngx_http_range_singlepart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) { @@ -707,7 +734,7 @@ ngx_http_range_singlepart_body(ngx_http_ out = NULL; ll = &out; - range = ctx->ranges.elts; + range = r->headers_out.ranges->elts; for (cl = in; cl; cl = cl->next) { @@ -787,96 +814,227 @@ static ngx_int_t ngx_http_range_multipart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) { - ngx_buf_t *b, *buf; - ngx_uint_t i; - ngx_chain_t *out, *hcl, *rcl, *dcl, **ll; - ngx_http_range_t *range; + off_t start, last, back; + ngx_buf_t *buf, *b; + ngx_uint_t i, finished; + ngx_chain_t *out, *cl, *ncl, **ll; + ngx_http_range_t *range, *tail; + + range = r->headers_out.ranges->elts; - ll = &out; - buf = in->buf; - range = ctx->ranges.elts; + if (!ctx->index) { + for (i = 0; i < r->headers_out.ranges->nelts; i++) { + if (ctx->offset < range[i].end) { + ctx->index = i + 1; + break; + } + } + } - for (i = 0; i < ctx->ranges.nelts; i++) { + tail = range + r->headers_out.ranges->nelts - 1; + range += ctx->index - 1; - /* - * The boundary header of the range: - * CRLF - * "--0123456789" CRLF - * "Content-Type: image/jpeg" CRLF - * "Content-Range: bytes " - */ + out = NULL; + ll = &out; + finished = 0; + + for (cl = in; cl; cl = cl->next) { + + buf = cl->buf; - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + start = ctx->offset; + last = ctx->offset + ngx_buf_size(buf); + + ctx->offset = last; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body buf: %O-%O", start, last); + + if (ngx_buf_special(buf)) { + *ll = cl; + ll = &cl->next; + continue; } - b->memory = 1; - b->pos = ctx->boundary_header.data; - b->last = ctx->boundary_header.data + ctx->boundary_header.len; + if (range->end <= start || range->start >= last) { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body skip"); - hcl = ngx_alloc_chain_link(r->pool); - if (hcl == NULL) { - return NGX_ERROR; + if (buf->in_file) { + buf->file_pos = buf->file_last; + } + + buf->pos = buf->last; + buf->sync = 1; + + continue; } - hcl->buf = b; + if (range->start >= start) { + if (ngx_http_range_link_boundary_header(r, ctx, &ll) != NGX_OK) { + return NGX_ERROR; + } - /* "SSSS-EEEE/TTTT" CRLF CRLF */ + if (buf->in_file) { + buf->file_pos += range->start - start; + } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + if (ngx_buf_in_memory(buf)) { + buf->pos += (size_t) (range->start - start); + } } - b->temporary = 1; - b->pos = range[i].content_range.data; - b->last = range[i].content_range.data + range[i].content_range.len; + if (range->end <= last) { + + if (range < tail && range[1].start < last) { + + b = ngx_alloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } + + ncl = ngx_alloc_chain_link(r->pool); + if (ncl == NULL) { + return NGX_ERROR; + } - rcl = ngx_alloc_chain_link(r->pool); - if (rcl == NULL) { - return NGX_ERROR; - } + ncl->buf = b; + ncl->next = cl; + + ngx_memcpy(b, buf, sizeof(ngx_buf_t)); + b->last_in_chain = 0; + b->last_buf = 0; + + back = last - range->end; + ctx->offset -= back; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body reuse buf: %O-%O", + ctx->offset, ctx->offset + back); - rcl->buf = b; + if (buf->in_file) { + buf->file_pos = buf->file_last - back; + } + + if (ngx_buf_in_memory(buf)) { + buf->pos = buf->last - back; + } + cl = ncl; + buf = cl->buf; + } + + if (buf->in_file) { + buf->file_last -= last - range->end; + } - /* the range data */ + if (ngx_buf_in_memory(buf)) { + buf->last -= (size_t) (last - range->end); + } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + if (range == tail) { + buf->last_buf = (r == r->main) ? 1 : 0; + buf->last_in_chain = 1; + *ll = cl; + ll = &cl->next; + + finished = 1; + break; + } + + range++; + ctx->index++; } - b->in_file = buf->in_file; - b->temporary = buf->temporary; - b->memory = buf->memory; - b->mmap = buf->mmap; - b->file = buf->file; + *ll = cl; + ll = &cl->next; + } + + if (out == NULL) { + return NGX_OK; + } + + *ll = NULL; + + if (finished + && ngx_http_range_link_last_boundary(r, ctx, ll) != NGX_OK) + { + return NGX_ERROR; + } + + return ngx_http_next_body_filter(r, out); +} + - if (buf->in_file) { - b->file_pos = buf->file_pos + range[i].start; - b->file_last = buf->file_pos + range[i].end; - } +static ngx_int_t +ngx_http_range_link_boundary_header(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t ***lll) +{ + ngx_buf_t *b; + ngx_chain_t *hcl, *rcl; + ngx_http_range_t *range; + + /* + * The boundary header of the range: + * CRLF + * "--0123456789" CRLF + * "Content-Type: image/jpeg" CRLF + * "Content-Range: bytes " + */ + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } - if (ngx_buf_in_memory(buf)) { - b->pos = buf->pos + (size_t) range[i].start; - b->last = buf->pos + (size_t) range[i].end; - } + b->memory = 1; + b->pos = ctx->boundary_header.data; + b->last = ctx->boundary_header.data + ctx->boundary_header.len; + + hcl = ngx_alloc_chain_link(r->pool); + if (hcl == NULL) { + return NGX_ERROR; + } + + hcl->buf = b; + + + /* "SSSS-EEEE/TTTT" CRLF CRLF */ + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } - dcl = ngx_alloc_chain_link(r->pool); - if (dcl == NULL) { - return NGX_ERROR; - } + range = r->headers_out.ranges->elts; + b->temporary = 1; + b->pos = range[ctx->index - 1].content_range.data; + b->last = range[ctx->index - 1].content_range.data + + range[ctx->index - 1].content_range.len; + + rcl = ngx_alloc_chain_link(r->pool); + if (rcl == NULL) { + return NGX_ERROR; + } + + rcl->buf = b; - dcl->buf = b; + **lll = hcl; + hcl->next = rcl; + *lll = &rcl->next; + + return NGX_OK; +} - *ll = hcl; - hcl->next = rcl; - rcl->next = dcl; - ll = &dcl->next; - } + +static ngx_int_t +ngx_http_range_link_last_boundary(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t **ll) +{ + ngx_buf_t *b; + ngx_chain_t *hcl; /* the last boundary CRLF "--0123456789--" CRLF */ @@ -886,7 +1044,8 @@ ngx_http_range_multipart_body(ngx_http_r } b->temporary = 1; - b->last_buf = 1; + b->last_in_chain = 1; + b->last_buf = (r == r->main) ? 1 : 0; b->pos = ngx_pnalloc(r->pool, sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof("--" CRLF) - 1); @@ -909,7 +1068,7 @@ ngx_http_range_multipart_body(ngx_http_r *ll = hcl; - return ngx_http_next_body_filter(r, out); + return NGX_OK; } diff -r b9850d3deb27 -r 62c100a0d426 src/http/ngx_http_request.h --- a/src/http/ngx_http_request.h Fri Oct 27 18:21:00 2017 +0800 +++ b/src/http/ngx_http_request.h Fri Oct 27 18:25:40 2017 +0800 @@ -251,6 +251,13 @@ typedef struct { typedef struct { + off_t start; + off_t end; + ngx_str_t content_range; +} ngx_http_range_t; + + +typedef struct { ngx_list_t headers; ngx_list_t trailers; @@ -278,6 +285,7 @@ typedef struct { u_char *content_type_lowcase; ngx_uint_t content_type_hash; + ngx_array_t *ranges; /* ngx_http_range_t */ ngx_array_t cache_control; off_t content_length_n; From hucong.c at foxmail.com Fri Oct 27 10:50:58 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Fri, 27 Oct 2017 18:50:58 +0800 Subject: [patch-2] Slice filter: support multiple ranges. Message-ID: # HG changeset patch # User hucongcong # Date 1509100127 -28800 # Fri Oct 27 18:28:47 2017 +0800 # Node ID 56fe1738d1b8bdfa59ecb3eb8934db6404061e47 # Parent 62c100a0d42614cd46f0719c0acb0ad914594217 Slice filter: support multiple ranges. diff -r 62c100a0d426 -r 56fe1738d1b8 src/http/modules/ngx_http_slice_filter_module.c --- a/src/http/modules/ngx_http_slice_filter_module.c Fri Oct 27 18:25:40 2017 +0800 +++ b/src/http/modules/ngx_http_slice_filter_module.c Fri Oct 27 18:28:47 2017 +0800 @@ -22,6 +22,8 @@ typedef struct { ngx_str_t etag; unsigned last:1; unsigned active:1; + unsigned multipart:1; + ngx_uint_t index; ngx_http_request_t *sr; } ngx_http_slice_ctx_t; @@ -103,7 +105,9 @@ ngx_http_slice_header_filter(ngx_http_re { off_t end; ngx_int_t rc; + ngx_uint_t i; ngx_table_elt_t *h; + ngx_http_range_t *range; ngx_http_slice_ctx_t *ctx; ngx_http_slice_loc_conf_t *slcf; ngx_http_slice_content_range_t cr; @@ -182,27 +186,48 @@ ngx_http_slice_header_filter(ngx_http_re r->allow_ranges = 1; r->subrequest_ranges = 1; - r->single_range = 1; rc = ngx_http_next_header_filter(r); - if (r != r->main) { - return rc; + if (r == r->main) { + r->preserve_body = 1; + + if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { + ctx->multipart = (r->headers_out.ranges->nelts != 1); + range = r->headers_out.ranges->elts; + + if (ctx->start + (off_t) slcf->size <= range[0].start) { + ctx->start = slcf->size * (range[0].start / slcf->size); + } + + ctx->end = range[r->headers_out.ranges->nelts - 1].end; + + } else { + ctx->end = cr.complete_length; + } } - r->preserve_body = 1; + if (ctx->multipart) { + end = ctx->start + (off_t) slcf->size - 1; + + range = r->headers_out.ranges->elts; + for (i = ctx->index; i < r->headers_out.ranges->nelts - 1; i++) { + + if (end < range[i].end) { + break; + } - if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { - if (ctx->start + (off_t) slcf->size <= r->headers_out.content_offset) { - ctx->start = slcf->size - * (r->headers_out.content_offset / slcf->size); + if (end < range[i + 1].start && ctx->start >= range[i].end) { + i++; + ctx->index = i; + ctx->start = slcf->size * (range[i].start / slcf->size); + + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "range multipart so fast forward to %O-%O @%O", + range[i].start, range[i].end, ctx->start); + break; + } } - - ctx->end = r->headers_out.content_offset - + r->headers_out.content_length_n; - - } else { - ctx->end = cr.complete_length; } return rc; From hungnv at opensource.com.vn Sat Oct 28 10:23:48 2017 From: hungnv at opensource.com.vn (Hung Nguyen) Date: Sat, 28 Oct 2017 17:23:48 +0700 Subject: [module] Tracking connection status from accepted to closed Message-ID: Hello, I am writing a HTTP core module that track http connection to know when it?s accepted and closed within nginx. What I am doing is register my module at PREACCESS_PHASE, and then adding a clean up callback to know when the connection closed (from nginx side). The problem is from my module i see nginx?s calling it twice, and calling my ngx_http_cleanup_t implement twice also. My question is: If I want to write my module to do exactly 2 things: 1. Know when connection accepted by nginx 2. and know when connection closed by nginx How should i implement my module? Thanks, From arut at nginx.com Mon Oct 30 11:49:04 2017 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 30 Oct 2017 14:49:04 +0300 Subject: [module] Tracking connection status from accepted to closed In-Reply-To: References: Message-ID: <20171030114904.GD75960@Romans-MacBook-Air.local> Hi, On Sat, Oct 28, 2017 at 05:23:48PM +0700, Hung Nguyen wrote: > Hello, > > I am writing a HTTP core module that track http connection to know when it?s accepted and closed within nginx. > > What I am doing is register my module at PREACCESS_PHASE, and then adding a clean up callback to know when the connection closed (from nginx side). > The problem is from my module i see nginx?s calling it twice, and calling my ngx_http_cleanup_t implement twice also. If you have an internal redirect in your code (for example when the index module is used), then after the redirect the request will pass again through the majority of phases, including the preaccess phase. > My question is: If I want to write my module to do exactly 2 things: > > 1. Know when connection accepted by nginx You can use the NGX_HTTP_POST_READ_PHASE phase instead of NGX_HTTP_PREACCESS_PHASE. Consider the ngx_http_realip_module as an example. > 2. and know when connection closed by nginx Use request pool cleanup - ngx_pool_cleanup_add(). > How should i implement my module? > > Thanks, > > > _______________________________________________ > nginx-devel mailing list > nginx-devel at nginx.org > http://mailman.nginx.org/mailman/listinfo/nginx-devel -- Roman Arutyunyan From arut at nginx.com Mon Oct 30 11:56:21 2017 From: arut at nginx.com (Roman Arutyunyan) Date: Mon, 30 Oct 2017 14:56:21 +0300 Subject: [module] Tracking connection status from accepted to closed In-Reply-To: <20171030114904.GD75960@Romans-MacBook-Air.local> References: <20171030114904.GD75960@Romans-MacBook-Air.local> Message-ID: <20171030115621.GE75960@Romans-MacBook-Air.local> On Mon, Oct 30, 2017 at 02:49:04PM +0300, Roman Arutyunyan wrote: > Hi, > > On Sat, Oct 28, 2017 at 05:23:48PM +0700, Hung Nguyen wrote: > > Hello, > > > > I am writing a HTTP core module that track http connection to know when it?s accepted and closed within nginx. > > > > What I am doing is register my module at PREACCESS_PHASE, and then adding a clean up callback to know when the connection closed (from nginx side). > > The problem is from my module i see nginx?s calling it twice, and calling my ngx_http_cleanup_t implement twice also. > > If you have an internal redirect in your code (for example when the index module > is used), then after the redirect the request will pass again through the > majority of phases, including the preaccess phase. > > > My question is: If I want to write my module to do exactly 2 things: > > > > 1. Know when connection accepted by nginx > > You can use the NGX_HTTP_POST_READ_PHASE phase instead of > NGX_HTTP_PREACCESS_PHASE. Consider the ngx_http_realip_module as an example. > > > 2. and know when connection closed by nginx > > Use request pool cleanup - ngx_pool_cleanup_add(). The above of course works for requests, not connections. However you can track connections in a similar way by registering a cleanup handler and searching it later from a phase handler. Take a look at the above mentioned realip module for details. -- Roman Arutyunyan From ben at isitdoneyet.co.uk Mon Oct 30 21:17:59 2017 From: ben at isitdoneyet.co.uk (Ben Brown) Date: Mon, 30 Oct 2017 21:17:59 +0000 Subject: [PATCH] Add 'log_index_denied' directive Message-ID: # HG changeset patch # User Ben Brown # Date 1509396532 0 # Mon Oct 30 20:48:52 2017 +0000 # Node ID 0c415222a6959147151422463261443275e69373 # Parent 9ef704d8563af4aff6817ab1c694fb40591f20b3 Add 'log_index_denied' directive This is similar to the 'log_not_found' directive but instead of suppressing 404 errors this can be used to suppress the 'index of directory...' error messages. It defaults to 'on', which is the current behaviour. It is valid in the same contexts as the 'log_not_found' directive. This was suggested by IRC user MacroMan to aid debugging where the logs contained a lot of these messages. diff -r 9ef704d8563a -r 0c415222a695 contrib/vim/syntax/nginx.vim --- a/contrib/vim/syntax/nginx.vim Tue Oct 17 19:52:16 2017 +0300 +++ b/contrib/vim/syntax/nginx.vim Mon Oct 30 20:48:52 2017 +0000 @@ -313,6 +313,7 @@ syn keyword ngxDirective contained load_module syn keyword ngxDirective contained lock_file syn keyword ngxDirective contained log_format +syn keyword ngxDirective contained log_index_denied syn keyword ngxDirective contained log_not_found syn keyword ngxDirective contained log_subrequest syn keyword ngxDirective contained map_hash_bucket_size diff -r 9ef704d8563a -r 0c415222a695 src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c Tue Oct 17 19:52:16 2017 +0300 +++ b/src/http/ngx_http_core_module.c Mon Oct 30 20:48:52 2017 +0000 @@ -583,6 +583,13 @@ offsetof(ngx_http_core_loc_conf_t, msie_refresh), NULL }, + { ngx_string("log_index_denied"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_core_loc_conf_t, log_index_denied), + NULL }, + { ngx_string("log_not_found"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, @@ -1153,9 +1160,10 @@ ngx_http_core_content_phase(ngx_http_request_t *r, ngx_http_phase_handler_t *ph) { - size_t root; - ngx_int_t rc; - ngx_str_t path; + size_t root; + ngx_int_t rc; + ngx_str_t path; + ngx_http_core_loc_conf_t *clcf; if (r->content_handler) { r->write_event_handler = ngx_http_request_empty_handler; @@ -1187,8 +1195,12 @@ if (r->uri.data[r->uri.len - 1] == '/') { if (ngx_http_map_uri_to_path(r, &path, &root, 0) != NULL) { - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "directory index of \"%s\" is forbidden", path.data); + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + if (clcf->log_index_denied) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "directory index of \"%s\" is forbidden", + path.data); + } } ngx_http_finalize_request(r, NGX_HTTP_FORBIDDEN); @@ -3384,6 +3396,7 @@ clcf->port_in_redirect = NGX_CONF_UNSET; clcf->msie_padding = NGX_CONF_UNSET; clcf->msie_refresh = NGX_CONF_UNSET; + clcf->log_index_denied = NGX_CONF_UNSET; clcf->log_not_found = NGX_CONF_UNSET; clcf->log_subrequest = NGX_CONF_UNSET; clcf->recursive_error_pages = NGX_CONF_UNSET; @@ -3649,6 +3662,7 @@ ngx_conf_merge_value(conf->port_in_redirect, prev->port_in_redirect, 1); ngx_conf_merge_value(conf->msie_padding, prev->msie_padding, 1); ngx_conf_merge_value(conf->msie_refresh, prev->msie_refresh, 0); + ngx_conf_merge_value(conf->log_index_denied, prev->log_index_denied, 1); ngx_conf_merge_value(conf->log_not_found, prev->log_not_found, 1); ngx_conf_merge_value(conf->log_subrequest, prev->log_subrequest, 0); ngx_conf_merge_value(conf->recursive_error_pages, diff -r 9ef704d8563a -r 0c415222a695 src/http/ngx_http_core_module.h --- a/src/http/ngx_http_core_module.h Tue Oct 17 19:52:16 2017 +0300 +++ b/src/http/ngx_http_core_module.h Mon Oct 30 20:48:52 2017 +0000 @@ -385,6 +385,7 @@ ngx_flag_t port_in_redirect; /* port_in_redirect */ ngx_flag_t msie_padding; /* msie_padding */ ngx_flag_t msie_refresh; /* msie_refresh */ + ngx_flag_t log_index_denied; /* log_index_denied */ ngx_flag_t log_not_found; /* log_not_found */ ngx_flag_t log_subrequest; /* log_subrequest */ ngx_flag_t recursive_error_pages; /* recursive_error_pages */ From hucong.c at foxmail.com Tue Oct 31 16:58:10 2017 From: hucong.c at foxmail.com (=?utf-8?B?6IOh6IGqIChodWNjKQ==?=) Date: Wed, 1 Nov 2017 00:58:10 +0800 Subject: [patch-2] Slice filter: support multiple ranges. In-Reply-To: References: Message-ID: Hi, Based on RFC 7233, I recently submitted a few patches. Any suggestion will be greatly appreciated. Here is a better patch. # HG changeset patch # User hucongcong # Date 1509100127 -28800 # Fri Oct 27 18:28:47 2017 +0800 # Node ID a9abe952df0f6fc5341fea766fb6543c38c65b67 # Parent 62c100a0d42614cd46f0719c0acb0ad914594217 Slice filter: support multiple ranges. diff -r 62c100a0d426 -r a9abe952df0f src/http/modules/ngx_http_slice_filter_module.c --- a/src/http/modules/ngx_http_slice_filter_module.c Fri Oct 27 18:25:40 2017 +0800 +++ b/src/http/modules/ngx_http_slice_filter_module.c Fri Oct 27 18:28:47 2017 +0800 @@ -22,6 +22,8 @@ typedef struct { ngx_str_t etag; unsigned last:1; unsigned active:1; + unsigned multipart:1; + ngx_uint_t index; ngx_http_request_t *sr; } ngx_http_slice_ctx_t; @@ -103,7 +105,9 @@ ngx_http_slice_header_filter(ngx_http_re { off_t end; ngx_int_t rc; + ngx_uint_t i; ngx_table_elt_t *h; + ngx_http_range_t *range; ngx_http_slice_ctx_t *ctx; ngx_http_slice_loc_conf_t *slcf; ngx_http_slice_content_range_t cr; @@ -182,27 +186,47 @@ ngx_http_slice_header_filter(ngx_http_re r->allow_ranges = 1; r->subrequest_ranges = 1; - r->single_range = 1; rc = ngx_http_next_header_filter(r); - if (r != r->main) { - return rc; + if (r == r->main) { + r->preserve_body = 1; + + if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { + ctx->multipart = (r->headers_out.ranges->nelts != 1); + range = r->headers_out.ranges->elts; + + if (ctx->start + (off_t) slcf->size <= range[0].start) { + ctx->start = slcf->size * (range[0].start / slcf->size); + } + + ctx->end = range[r->headers_out.ranges->nelts - 1].end; + + } else { + ctx->end = cr.complete_length; + } } - r->preserve_body = 1; + if (ctx->multipart) { + range = r->headers_out.ranges->elts; + + for (i = ctx->index; i < r->headers_out.ranges->nelts - 1; i++) { + + if (ctx->start < range[i].end) { + break; + } - if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { - if (ctx->start + (off_t) slcf->size <= r->headers_out.content_offset) { - ctx->start = slcf->size - * (r->headers_out.content_offset / slcf->size); + if (ctx->start + (off_t) slcf->size <= range[i + 1].start) { + i++; + ctx->index = i; + ctx->start = slcf->size * (range[i].start / slcf->size); + + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "range multipart so fast forward to %O-%O @%O", + range[i].start, range[i].end, ctx->start); + break; + } } - - ctx->end = r->headers_out.content_offset - + r->headers_out.content_length_n; - - } else { - ctx->end = cr.complete_length; } return rc; ------------------ Original ------------------ From: "?? (hucc)";; Date: Oct 27, 2017 To: "nginx-devel"; Subject: [patch-2] Slice filter: support multiple ranges. # HG changeset patch # User hucongcong # Date 1509100127 -28800 # Fri Oct 27 18:28:47 2017 +0800 # Node ID 56fe1738d1b8bdfa59ecb3eb8934db6404061e47 # Parent 62c100a0d42614cd46f0719c0acb0ad914594217 Slice filter: support multiple ranges. diff -r 62c100a0d426 -r 56fe1738d1b8 src/http/modules/ngx_http_slice_filter_module.c --- a/src/http/modules/ngx_http_slice_filter_module.c Fri Oct 27 18:25:40 2017 +0800 +++ b/src/http/modules/ngx_http_slice_filter_module.c Fri Oct 27 18:28:47 2017 +0800 @@ -22,6 +22,8 @@ typedef struct { ngx_str_t etag; unsigned last:1; unsigned active:1; + unsigned multipart:1; + ngx_uint_t index; ngx_http_request_t *sr; } ngx_http_slice_ctx_t; @@ -103,7 +105,9 @@ ngx_http_slice_header_filter(ngx_http_re { off_t end; ngx_int_t rc; + ngx_uint_t i; ngx_table_elt_t *h; + ngx_http_range_t *range; ngx_http_slice_ctx_t *ctx; ngx_http_slice_loc_conf_t *slcf; ngx_http_slice_content_range_t cr; @@ -182,27 +186,48 @@ ngx_http_slice_header_filter(ngx_http_re r->allow_ranges = 1; r->subrequest_ranges = 1; - r->single_range = 1; rc = ngx_http_next_header_filter(r); - if (r != r->main) { - return rc; + if (r == r->main) { + r->preserve_body = 1; + + if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { + ctx->multipart = (r->headers_out.ranges->nelts != 1); + range = r->headers_out.ranges->elts; + + if (ctx->start + (off_t) slcf->size <= range[0].start) { + ctx->start = slcf->size * (range[0].start / slcf->size); + } + + ctx->end = range[r->headers_out.ranges->nelts - 1].end; + + } else { + ctx->end = cr.complete_length; + } } - r->preserve_body = 1; + if (ctx->multipart) { + end = ctx->start + (off_t) slcf->size - 1; + + range = r->headers_out.ranges->elts; + for (i = ctx->index; i < r->headers_out.ranges->nelts - 1; i++) { + + if (end < range[i].end) { + break; + } - if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { - if (ctx->start + (off_t) slcf->size <= r->headers_out.content_offset) { - ctx->start = slcf->size - * (r->headers_out.content_offset / slcf->size); + if (end < range[i + 1].start && ctx->start >= range[i].end) { + i++; + ctx->index = i; + ctx->start = slcf->size * (range[i].start / slcf->size); + + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "range multipart so fast forward to %O-%O @%O", + range[i].start, range[i].end, ctx->start); + break; + } } - - ctx->end = r->headers_out.content_offset - + r->headers_out.content_length_n; - - } else { - ctx->end = cr.complete_length; } return rc; _______________________________________________ nginx-devel mailing list nginx-devel at nginx.org http://mailman.nginx.org/mailman/listinfo/nginx-devel