From ru at nginx.com  Tue Aug  1 11:36:20 2017
From: ru at nginx.com (Ruslan Ermilov)
Date: Tue, 01 Aug 2017 11:36:20 +0000
Subject: [nginx] Browser: style.
Message-ID: <hg.1b82e86dd3e7.1501587380.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/1b82e86dd3e7
branches:  
changeset: 7076:1b82e86dd3e7
user:      Ruslan Ermilov <ru at nginx.com>
date:      Tue Aug 01 14:27:23 2017 +0300
description:
Browser: style.

Removed custom variable type and renamed function that adds variables.

diffstat:

 src/http/modules/ngx_http_browser_module.c |  45 ++++++++++++++----------------
 1 files changed, 21 insertions(+), 24 deletions(-)

diffs (89 lines):

diff -r 72d3aefc2993 -r 1b82e86dd3e7 src/http/modules/ngx_http_browser_module.c
--- a/src/http/modules/ngx_http_browser_module.c	Wed Jul 26 13:13:51 2017 +0300
+++ b/src/http/modules/ngx_http_browser_module.c	Tue Aug 01 14:27:23 2017 +0300
@@ -38,13 +38,6 @@ typedef struct {
 
 
 typedef struct {
-    ngx_str_t                   name;
-    ngx_http_get_variable_pt    handler;
-    uintptr_t                   data;
-} ngx_http_browser_variable_t;
-
-
-typedef struct {
     ngx_array_t                *modern_browsers;
     ngx_array_t                *ancient_browsers;
     ngx_http_variable_value_t  *modern_browser_value;
@@ -63,7 +56,7 @@ static ngx_int_t ngx_http_browser_variab
 static ngx_uint_t ngx_http_browser(ngx_http_request_t *r,
     ngx_http_browser_conf_t *cf);
 
-static ngx_int_t ngx_http_browser_add_variable(ngx_conf_t *cf);
+static ngx_int_t ngx_http_browser_add_variables(ngx_conf_t *cf);
 static void *ngx_http_browser_create_conf(ngx_conf_t *cf);
 static char *ngx_http_browser_merge_conf(ngx_conf_t *cf, void *parent,
     void *child);
@@ -114,7 +107,7 @@ static ngx_command_t  ngx_http_browser_c
 
 
 static ngx_http_module_t  ngx_http_browser_module_ctx = {
-    ngx_http_browser_add_variable,         /* preconfiguration */
+    ngx_http_browser_add_variables,        /* preconfiguration */
     NULL,                                  /* postconfiguration */
 
     NULL,                                  /* create main configuration */
@@ -218,13 +211,18 @@ static ngx_http_modern_browser_mask_t  n
 };
 
 
-static ngx_http_browser_variable_t  ngx_http_browsers[] = {
-    { ngx_string("msie"), ngx_http_msie_variable, 0 },
-    { ngx_string("modern_browser"), ngx_http_browser_variable,
-          NGX_HTTP_MODERN_BROWSER },
-    { ngx_string("ancient_browser"), ngx_http_browser_variable,
-          NGX_HTTP_ANCIENT_BROWSER },
-    { ngx_null_string, NULL, 0 }
+static ngx_http_variable_t  ngx_http_browser_vars[] = {
+
+    { ngx_string("msie"), NULL, ngx_http_msie_variable,
+      0, NGX_HTTP_VAR_CHANGEABLE, 0 },
+
+    { ngx_string("modern_browser"), NULL, ngx_http_browser_variable,
+      NGX_HTTP_MODERN_BROWSER, NGX_HTTP_VAR_CHANGEABLE, 0 },
+
+    { ngx_string("ancient_browser"), NULL, ngx_http_browser_variable,
+      NGX_HTTP_ANCIENT_BROWSER, NGX_HTTP_VAR_CHANGEABLE, 0 },
+
+    { ngx_null_string, NULL, NULL, 0, 0, 0 }
 };
 
 
@@ -397,20 +395,19 @@ ngx_http_msie_variable(ngx_http_request_
 
 
 static ngx_int_t
-ngx_http_browser_add_variable(ngx_conf_t *cf)
+ngx_http_browser_add_variables(ngx_conf_t *cf)
 {
-    ngx_http_browser_variable_t   *var;
-    ngx_http_variable_t           *v;
+    ngx_http_variable_t  *var, *v;
 
-    for (var = ngx_http_browsers; var->name.len; var++) {
+    for (v = ngx_http_browser_vars; v->name.len; v++) {
 
-        v = ngx_http_add_variable(cf, &var->name, NGX_HTTP_VAR_CHANGEABLE);
-        if (v == NULL) {
+        var = ngx_http_add_variable(cf, &v->name, v->flags);
+        if (var == NULL) {
             return NGX_ERROR;
         }
 
-        v->get_handler = var->handler;
-        v->data = var->data;
+        var->get_handler = v->get_handler;
+        var->data = v->data;
     }
 
     return NGX_OK;

From ru at nginx.com  Tue Aug  1 11:36:23 2017
From: ru at nginx.com (Ruslan Ermilov)
Date: Tue, 01 Aug 2017 11:36:23 +0000
Subject: [nginx] Variables: macros for null variables.
Message-ID: <hg.2a288909abc6.1501587383.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/2a288909abc6
branches:  
changeset: 7077:2a288909abc6
user:      Ruslan Ermilov <ru at nginx.com>
date:      Tue Aug 01 14:28:33 2017 +0300
description:
Variables: macros for null variables.

No functional changes.

diffstat:

 src/http/modules/ngx_http_browser_module.c     |  2 +-
 src/http/modules/ngx_http_fastcgi_module.c     |  2 +-
 src/http/modules/ngx_http_geoip_module.c       |  2 +-
 src/http/modules/ngx_http_proxy_module.c       |  2 +-
 src/http/modules/ngx_http_realip_module.c      |  2 +-
 src/http/modules/ngx_http_ssi_filter_module.c  |  2 +-
 src/http/modules/ngx_http_ssl_module.c         |  2 +-
 src/http/modules/ngx_http_stub_status_module.c |  2 +-
 src/http/ngx_http_upstream.c                   |  2 +-
 src/http/ngx_http_variables.c                  |  2 +-
 src/http/ngx_http_variables.h                  |  2 ++
 src/http/v2/ngx_http_v2_module.c               |  2 +-
 src/stream/ngx_stream_geoip_module.c           |  2 +-
 src/stream/ngx_stream_realip_module.c          |  2 +-
 src/stream/ngx_stream_ssl_module.c             |  2 +-
 src/stream/ngx_stream_ssl_preread_module.c     |  2 +-
 src/stream/ngx_stream_upstream.c               |  2 +-
 src/stream/ngx_stream_variables.c              |  2 +-
 src/stream/ngx_stream_variables.h              |  2 ++
 19 files changed, 21 insertions(+), 17 deletions(-)

diffs (228 lines):

diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/modules/ngx_http_browser_module.c
--- a/src/http/modules/ngx_http_browser_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/modules/ngx_http_browser_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -222,7 +222,7 @@ static ngx_http_variable_t  ngx_http_bro
     { ngx_string("ancient_browser"), NULL, ngx_http_browser_variable,
       NGX_HTTP_ANCIENT_BROWSER, NGX_HTTP_VAR_CHANGEABLE, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/modules/ngx_http_fastcgi_module.c
--- a/src/http/modules/ngx_http_fastcgi_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/modules/ngx_http_fastcgi_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -631,7 +631,7 @@ static ngx_http_variable_t  ngx_http_fas
       ngx_http_fastcgi_path_info_variable, 0,
       NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/modules/ngx_http_geoip_module.c
--- a/src/http/modules/ngx_http_geoip_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/modules/ngx_http_geoip_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -232,7 +232,7 @@ static ngx_http_variable_t  ngx_http_geo
       ngx_http_geoip_city_int_variable,
       offsetof(GeoIPRecord, area_code), 0, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/modules/ngx_http_proxy_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -829,7 +829,7 @@ static ngx_http_variable_t  ngx_http_pro
       ngx_http_proxy_internal_chunked_variable, 0,
       NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/modules/ngx_http_realip_module.c
--- a/src/http/modules/ngx_http_realip_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/modules/ngx_http_realip_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -122,7 +122,7 @@ static ngx_http_variable_t  ngx_http_rea
     { ngx_string("realip_remote_port"), NULL,
       ngx_http_realip_remote_port_variable, 0, 0, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/modules/ngx_http_ssi_filter_module.c
--- a/src/http/modules/ngx_http_ssi_filter_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/modules/ngx_http_ssi_filter_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -321,7 +321,7 @@ static ngx_http_variable_t  ngx_http_ssi
     { ngx_string("date_gmt"), NULL, ngx_http_ssi_date_gmt_local_variable, 1,
       NGX_HTTP_VAR_NOCACHEABLE, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/modules/ngx_http_ssl_module.c
--- a/src/http/modules/ngx_http_ssl_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/modules/ngx_http_ssl_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -329,7 +329,7 @@ static ngx_http_variable_t  ngx_http_ssl
     { ngx_string("ssl_client_v_remain"), NULL, ngx_http_ssl_variable,
       (uintptr_t) ngx_ssl_get_client_v_remain, NGX_HTTP_VAR_CHANGEABLE, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/modules/ngx_http_stub_status_module.c
--- a/src/http/modules/ngx_http_stub_status_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/modules/ngx_http_stub_status_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -76,7 +76,7 @@ static ngx_http_variable_t  ngx_http_stu
     { ngx_string("connections_waiting"), NULL, ngx_http_stub_status_variable,
       3, NGX_HTTP_VAR_NOCACHEABLE, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/ngx_http_upstream.c	Tue Aug 01 14:28:33 2017 +0300
@@ -427,7 +427,7 @@ static ngx_http_variable_t  ngx_http_ups
     { ngx_string("upstream_cookie_"), NULL, ngx_http_upstream_cookie_variable,
       0, NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_PREFIX, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/ngx_http_variables.c
--- a/src/http/ngx_http_variables.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/ngx_http_variables.c	Tue Aug 01 14:28:33 2017 +0300
@@ -376,7 +376,7 @@ static ngx_http_variable_t  ngx_http_cor
     { ngx_string("arg_"), NULL, ngx_http_variable_argument,
       0, NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_PREFIX, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/ngx_http_variables.h
--- a/src/http/ngx_http_variables.h	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/ngx_http_variables.h	Tue Aug 01 14:28:33 2017 +0300
@@ -43,6 +43,8 @@ struct ngx_http_variable_s {
     ngx_uint_t                    index;
 };
 
+#define ngx_http_null_variable  { ngx_null_string, NULL, NULL, 0, 0, 0 }
+
 
 ngx_http_variable_t *ngx_http_add_variable(ngx_conf_t *cf, ngx_str_t *name,
     ngx_uint_t flags);
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/http/v2/ngx_http_v2_module.c
--- a/src/http/v2/ngx_http_v2_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/http/v2/ngx_http_v2_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -225,7 +225,7 @@ static ngx_http_variable_t  ngx_http_v2_
     { ngx_string("http2"), NULL,
       ngx_http_v2_variable, 0, 0, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_http_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/stream/ngx_stream_geoip_module.c
--- a/src/stream/ngx_stream_geoip_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/stream/ngx_stream_geoip_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -210,7 +210,7 @@ static ngx_stream_variable_t  ngx_stream
       ngx_stream_geoip_city_int_variable,
       offsetof(GeoIPRecord, area_code), 0, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_stream_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/stream/ngx_stream_realip_module.c
--- a/src/stream/ngx_stream_realip_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/stream/ngx_stream_realip_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -89,7 +89,7 @@ static ngx_stream_variable_t  ngx_stream
     { ngx_string("realip_remote_port"), NULL,
       ngx_stream_realip_remote_port_variable, 0, 0, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_stream_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/stream/ngx_stream_ssl_module.c
--- a/src/stream/ngx_stream_ssl_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/stream/ngx_stream_ssl_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -273,7 +273,7 @@ static ngx_stream_variable_t  ngx_stream
     { ngx_string("ssl_client_v_remain"), NULL, ngx_stream_ssl_variable,
       (uintptr_t) ngx_ssl_get_client_v_remain, NGX_STREAM_VAR_CHANGEABLE, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_stream_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/stream/ngx_stream_ssl_preread_module.c
--- a/src/stream/ngx_stream_ssl_preread_module.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/stream/ngx_stream_ssl_preread_module.c	Tue Aug 01 14:28:33 2017 +0300
@@ -85,7 +85,7 @@ static ngx_stream_variable_t  ngx_stream
     { ngx_string("ssl_preread_server_name"), NULL,
       ngx_stream_ssl_preread_server_name_variable, 0, 0, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_stream_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/stream/ngx_stream_upstream.c
--- a/src/stream/ngx_stream_upstream.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/stream/ngx_stream_upstream.c	Tue Aug 01 14:28:33 2017 +0300
@@ -100,7 +100,7 @@ static ngx_stream_variable_t  ngx_stream
       ngx_stream_upstream_bytes_variable, 1,
       NGX_STREAM_VAR_NOCACHEABLE, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_stream_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/stream/ngx_stream_variables.c
--- a/src/stream/ngx_stream_variables.c	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/stream/ngx_stream_variables.c	Tue Aug 01 14:28:33 2017 +0300
@@ -111,7 +111,7 @@ static ngx_stream_variable_t  ngx_stream
     { ngx_string("protocol"), NULL,
       ngx_stream_variable_protocol, 0, 0, 0 },
 
-    { ngx_null_string, NULL, NULL, 0, 0, 0 }
+      ngx_stream_null_variable
 };
 
 
diff -r 1b82e86dd3e7 -r 2a288909abc6 src/stream/ngx_stream_variables.h
--- a/src/stream/ngx_stream_variables.h	Tue Aug 01 14:27:23 2017 +0300
+++ b/src/stream/ngx_stream_variables.h	Tue Aug 01 14:28:33 2017 +0300
@@ -43,6 +43,8 @@ struct ngx_stream_variable_s {
     ngx_uint_t                    index;
 };
 
+#define ngx_stream_null_variable  { ngx_null_string, NULL, NULL, 0, 0, 0 }
+
 
 ngx_stream_variable_t *ngx_stream_add_variable(ngx_conf_t *cf, ngx_str_t *name,
     ngx_uint_t flags);

From ru at nginx.com  Tue Aug  1 16:13:12 2017
From: ru at nginx.com (Ruslan Ermilov)
Date: Tue, 01 Aug 2017 16:13:12 +0000
Subject: [nginx] Upstream zone: store peers->name and its data in shared
 memory.
Message-ID: <hg.1eb753aa8e5e.1501603992.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/1eb753aa8e5e
branches:  
changeset: 7078:1eb753aa8e5e
user:      Ruslan Ermilov <ru at nginx.com>
date:      Tue Aug 01 19:12:10 2017 +0300
description:
Upstream zone: store peers->name and its data in shared memory.

The shared objects should generally be allocated from shared memory.
While peers->name and the data it points to allocated from cf->pool
happened to work on UNIX, it broke on Windows.  On UNIX this worked
only because the shared memory zone for upstreams is re-created for
every new configuration.

But on Windows, a worker process does not inherit the address space
of the master process, so the peers->name pointed to data allocated
from cf->pool by the master process, and was invalid.

diffstat:

 src/http/modules/ngx_http_upstream_zone_module.c |  18 ++++++++++++++++++
 src/stream/ngx_stream_upstream_zone_module.c     |  18 ++++++++++++++++++
 2 files changed, 36 insertions(+), 0 deletions(-)

diffs (84 lines):

diff -r 2a288909abc6 -r 1eb753aa8e5e src/http/modules/ngx_http_upstream_zone_module.c
--- a/src/http/modules/ngx_http_upstream_zone_module.c	Tue Aug 01 14:28:33 2017 +0300
+++ b/src/http/modules/ngx_http_upstream_zone_module.c	Tue Aug 01 19:12:10 2017 +0300
@@ -185,6 +185,7 @@ static ngx_http_upstream_rr_peers_t *
 ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
     ngx_http_upstream_srv_conf_t *uscf)
 {
+    ngx_str_t                     *name;
     ngx_http_upstream_rr_peer_t   *peer, **peerp;
     ngx_http_upstream_rr_peers_t  *peers, *backup;
 
@@ -195,6 +196,21 @@ ngx_http_upstream_zone_copy_peers(ngx_sl
 
     ngx_memcpy(peers, uscf->peer.data, sizeof(ngx_http_upstream_rr_peers_t));
 
+    name = ngx_slab_alloc(shpool, sizeof(ngx_str_t));
+    if (name == NULL) {
+        return NULL;
+    }
+
+    name->data = ngx_slab_alloc(shpool, peers->name->len);
+    if (name->data == NULL) {
+        return NULL;
+    }
+
+    ngx_memcpy(name->data, peers->name->data, peers->name->len);
+    name->len = peers->name->len;
+
+    peers->name = name;
+
     peers->shpool = shpool;
 
     for (peerp = &peers->peer; *peerp; peerp = &peer->next) {
@@ -221,6 +237,8 @@ ngx_http_upstream_zone_copy_peers(ngx_sl
 
     ngx_memcpy(backup, peers->next, sizeof(ngx_http_upstream_rr_peers_t));
 
+    backup->name = name;
+
     backup->shpool = shpool;
 
     for (peerp = &backup->peer; *peerp; peerp = &peer->next) {
diff -r 2a288909abc6 -r 1eb753aa8e5e src/stream/ngx_stream_upstream_zone_module.c
--- a/src/stream/ngx_stream_upstream_zone_module.c	Tue Aug 01 14:28:33 2017 +0300
+++ b/src/stream/ngx_stream_upstream_zone_module.c	Tue Aug 01 19:12:10 2017 +0300
@@ -182,6 +182,7 @@ static ngx_stream_upstream_rr_peers_t *
 ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
     ngx_stream_upstream_srv_conf_t *uscf)
 {
+    ngx_str_t                       *name;
     ngx_stream_upstream_rr_peer_t   *peer, **peerp;
     ngx_stream_upstream_rr_peers_t  *peers, *backup;
 
@@ -192,6 +193,21 @@ ngx_stream_upstream_zone_copy_peers(ngx_
 
     ngx_memcpy(peers, uscf->peer.data, sizeof(ngx_stream_upstream_rr_peers_t));
 
+    name = ngx_slab_alloc(shpool, sizeof(ngx_str_t));
+    if (name == NULL) {
+        return NULL;
+    }
+
+    name->data = ngx_slab_alloc(shpool, peers->name->len);
+    if (name->data == NULL) {
+        return NULL;
+    }
+
+    ngx_memcpy(name->data, peers->name->data, peers->name->len);
+    name->len = peers->name->len;
+
+    peers->name = name;
+
     peers->shpool = shpool;
 
     for (peerp = &peers->peer; *peerp; peerp = &peer->next) {
@@ -218,6 +234,8 @@ ngx_stream_upstream_zone_copy_peers(ngx_
 
     ngx_memcpy(backup, peers->next, sizeof(ngx_stream_upstream_rr_peers_t));
 
+    backup->name = name;
+
     backup->shpool = shpool;
 
     for (peerp = &backup->peer; *peerp; peerp = &peer->next) {

From m at rkellis.com  Wed Aug  2 11:30:11 2017
From: m at rkellis.com (Mark Ellis)
Date: Wed, 02 Aug 2017 11:30:11 +0000
Subject: [PATCH] Allows hyphens in variable names so can access hyphenated
 query vars
Message-ID: <08008b0d1f4ffbd017d9.1501673411@ma.h1.thisaintnews.com>

# HG changeset patch
# User Mark Ellis <m at rkellis.com>
# Date 1501670303 -3600
#      Wed Aug 02 11:38:23 2017 +0100
# Node ID 08008b0d1f4ffbd017d9f50142bdd46d7c4a41d2
# Parent  1eb753aa8e5e9ee5059ebb544194c8919b674873
Allows hyphens in variable names so can access hyphenated query vars

This patch allows hyphens in variable names so that you can access query string
parameters with hyphens

Before if you had a query string param such as `?foo-bar=123` you could not
access it with the variable `$foo-bar` as variables could not contain hyphens.

diff -r 1eb753aa8e5e -r 08008b0d1f4f src/http/ngx_http_script.c
--- a/src/http/ngx_http_script.c	Tue Aug 01 19:12:10 2017 +0300
+++ b/src/http/ngx_http_script.c	Wed Aug 02 11:38:23 2017 +0100
@@ -403,6 +403,7 @@
                 if ((ch >= 'A' && ch <= 'Z')
                     || (ch >= 'a' && ch <= 'z')
                     || (ch >= '0' && ch <= '9')
+                    || ch == '-'
                     || ch == '_')
                 {
                     continue;

From rmahindra at uber.com  Wed Aug  2 17:43:05 2017
From: rmahindra at uber.com (Rajesh Mahindra)
Date: Wed, 2 Aug 2017 10:43:05 -0700
Subject: UDP proxy
Message-ID: <CAMPitreRrtZ_sP46=D1C+w35StMB0JoSB+EFa9Q=2xHHgbkCtg@mail.gmail.com>

Hi

I am trying to setup nginx as a UDP load balancer using
ngx_stream_core_module to load balance udp traffic across backend
instances.

However, i see that for every incoming UDP packet from a client, nginx uses
a different source port when sending the packet to the backend instance for
the same UDP connection. That is throwing the backend off. I dont want to
use nginx as a transparent proxy to avoid setting up routes (since i am
deploying on Google Cloud). Is there a way nginx can load balancer based on
the UDP connection?

I am using the following config:

upstream test_upstreams {

        server 104.199.242.111:6121;

        server 104.199.242.111:6122;

        hash $remote_addr consistent;

    }


    server {

        listen 443 udp;

        preread_buffer_size 1500;

        proxy_pass test_upstreams;

        proxy_timeout 10s;

        proxy_responses 1;

        access_log logs/quic_access.log quic;

        error_log logs/udp.log;

    }


Thanks
Rajesh
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170802/fef31811/attachment.html>

From chengwenping1 at jd.com  Thu Aug  3 02:01:09 2017
From: chengwenping1 at jd.com (=?utf-8?B?56iL5paH5bmz?=)
Date: Thu, 3 Aug 2017 02:01:09 +0000
Subject: =?UTF-8?B?562U5aSNOiBVRFAgcHJveHk=?=
In-Reply-To: <CAMPitreRrtZ_sP46=D1C+w35StMB0JoSB+EFa9Q=2xHHgbkCtg@mail.gmail.com>
References: <CAMPitreRrtZ_sP46=D1C+w35StMB0JoSB+EFa9Q=2xHHgbkCtg@mail.gmail.com>
Message-ID: <31F771DF13463A429610AEEBF6AFAE82017B9B56@mbx14.360buyAD.local>

Hi,
         I have open a ticket for this issue in https://trac.nginx.org/nginx/ticket/1298, but from the response, there is no perfect solution for it.

         Thanks,
         Nick
???: nginx-devel [mailto:nginx-devel-bounces at nginx.org] ?? Rajesh Mahindra via nginx-devel
????: 2017?8?3? 1:43
???: nginx-devel at nginx.org
??: Rajesh Mahindra
??: UDP proxy

Hi

I am trying to setup nginx as a UDP load balancer using ngx_stream_core_module to load balance udp traffic across backend instances.

However, i see that for every incoming UDP packet from a client, nginx uses a different source port when sending the packet to the backend instance for the same UDP connection. That is throwing the backend off. I dont want to use nginx as a transparent proxy to avoid setting up routes (since i am deploying on Google Cloud). Is there a way nginx can load balancer based on the UDP connection?

I am using the following config:


upstream test_upstreams {

        server 104.199.242.111:6121<http://104.199.242.111:6121>;

        server 104.199.242.111:6122<http://104.199.242.111:6122>;

        hash $remote_addr consistent;

    }



    server {

        listen 443 udp;

        preread_buffer_size 1500;

        proxy_pass test_upstreams;

        proxy_timeout 10s;

        proxy_responses 1;

        access_log logs/quic_access.log quic;

        error_log logs/udp.log;

    }


Thanks
Rajesh
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170803/9dd7a79e/attachment.html>

From ru at nginx.com  Fri Aug  4 05:03:18 2017
From: ru at nginx.com (Ruslan Ermilov)
Date: Fri, 04 Aug 2017 05:03:18 +0000
Subject: [nginx] Referer: fixed $invalid_referer.
Message-ID: <hg.7564a919d333.1501822998.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/7564a919d333
branches:  
changeset: 7079:7564a919d333
user:      Ruslan Ermilov <ru at nginx.com>
date:      Fri Aug 04 08:01:55 2017 +0300
description:
Referer: fixed $invalid_referer.

The variable was considered non-existent in the absence of any
valid_referers directives.

Given the following config snippet,

    location / {
        return 200 $invalid_referer;
    }

    location /referer {
        valid_referers server_names;
    }

"location /" should work identically and independently on other
"location /referer".

The fix is to always add the $invalid_referer variable as long
as the module is compiled in, as is done by other modules.

diffstat:

 src/http/modules/ngx_http_referer_module.c |  39 +++++++++++++++++++----------
 1 files changed, 25 insertions(+), 14 deletions(-)

diffs (77 lines):

diff -r 1eb753aa8e5e -r 7564a919d333 src/http/modules/ngx_http_referer_module.c
--- a/src/http/modules/ngx_http_referer_module.c	Tue Aug 01 19:12:10 2017 +0300
+++ b/src/http/modules/ngx_http_referer_module.c	Fri Aug 04 08:01:55 2017 +0300
@@ -32,6 +32,7 @@ typedef struct {
 } ngx_http_referer_conf_t;
 
 
+static ngx_int_t ngx_http_referer_add_variables(ngx_conf_t *cf);
 static void * ngx_http_referer_create_conf(ngx_conf_t *cf);
 static char * ngx_http_referer_merge_conf(ngx_conf_t *cf, void *parent,
     void *child);
@@ -77,7 +78,7 @@ static ngx_command_t  ngx_http_referer_c
 
 
 static ngx_http_module_t  ngx_http_referer_module_ctx = {
-    NULL,                                  /* preconfiguration */
+    ngx_http_referer_add_variables,        /* preconfiguration */
     NULL,                                  /* postconfiguration */
 
     NULL,                                  /* create main configuration */
@@ -107,6 +108,9 @@ ngx_module_t  ngx_http_referer_module = 
 };
 
 
+static ngx_str_t  ngx_http_invalid_referer_name = ngx_string("invalid_referer");
+
+
 static ngx_int_t
 ngx_http_referer_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v,
     uintptr_t data)
@@ -263,6 +267,23 @@ valid:
 }
 
 
+static ngx_int_t
+ngx_http_referer_add_variables(ngx_conf_t *cf)
+{
+    ngx_http_variable_t  *var;
+
+    var = ngx_http_add_variable(cf, &ngx_http_invalid_referer_name,
+                                NGX_HTTP_VAR_CHANGEABLE);
+    if (var == NULL) {
+        return NGX_ERROR;
+    }
+
+    var->get_handler = ngx_http_referer_variable;
+
+    return NGX_OK;
+}
+
+
 static void *
 ngx_http_referer_create_conf(ngx_conf_t *cf)
 {
@@ -452,19 +473,9 @@ ngx_http_valid_referers(ngx_conf_t *cf, 
 {
     ngx_http_referer_conf_t  *rlcf = conf;
 
-    u_char                    *p;
-    ngx_str_t                 *value, uri, name;
-    ngx_uint_t                 i;
-    ngx_http_variable_t       *var;
-
-    ngx_str_set(&name, "invalid_referer");
-
-    var = ngx_http_add_variable(cf, &name, NGX_HTTP_VAR_CHANGEABLE);
-    if (var == NULL) {
-        return NGX_CONF_ERROR;
-    }
-
-    var->get_handler = ngx_http_referer_variable;
+    u_char      *p;
+    ngx_str_t   *value, uri;
+    ngx_uint_t   i;
 
     if (rlcf->keys == NULL) {
         rlcf->keys = ngx_pcalloc(cf->temp_pool, sizeof(ngx_hash_keys_arrays_t));

From fangpeng1986 at gmail.com  Fri Aug  4 07:22:29 2017
From: fangpeng1986 at gmail.com (Peng Fang)
Date: Fri, 4 Aug 2017 15:22:29 +0800
Subject: proxy_cache_background_update and slice filter module
Message-ID: <CALH_F=bf0whdEfjHR8W0j0bGOHnBjpPedbR2vbC5Ku7mY29x0A@mail.gmail.com>

Hello,

It seems that proxy_cache_background_update  and slice filter module do not
cooperate well.
when proxy_cache_background_update and slice are both enabled, the variable
of upstream_cache_status will always be "STALE" rather than getting back to
"HIT" after the revalidation.

Thanks.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170804/1eded79f/attachment.html>

From niew at mcmaster.ca  Sun Aug  6 00:44:53 2017
From: niew at mcmaster.ca (Wayde Nie)
Date: Sat, 5 Aug 2017 20:44:53 -0400
Subject: [PATCH] Add proxy_protocol option to mail listener
Message-ID: <ca09cf40-7f60-e63a-8957-d14f0bb297d6@mcmaster.ca>

Hi Kees, Maxim,

I'm very interested in proxy_protocol enabled nginx mail proxy as well.
Wondering if the feature might be more straightforward (initially?) if,
when enabled on the listen line, it simply set the appropriate
$proxy_protocol_* variables, similar to the http servers?

I was hoping that I could then include it as an http get param in the
url for my auth_http directive for the auth script to do a dns blacklist
lookup and/or logging (and possibly some other anti spambot efforts).

I'm working on it with your patch as a starting point, and I can get it
to compile cleanly.  Nginx keeps working as expected when I don't set
proxy_protocol on the listen directive, and when I do set proxy_protocol
on the listen directive I see the correct ip and port picked up and
logged in the error.log, however, then nginx stops sending the smtp
greeting... My mail client connects to my loadbalancer, the lb connects
to nginx:587 sending the PROXY line, nginx parses and logs the PROXY
fields,  then the client times out waiting for any return traffic from
nginx... I know it's something I'm doing :-)

I'm happy to keep poking away at it, but curious mostly, if you think
the approach is sound? (ie. use $proxy_protocol_addr, set by
proxy_protocol directive and pass in to auth_http script in auth url as
a get param?) and if an initial patch that starts by just setting
$proxy_protocol_* variables would be a useful first step in this type of
functionality?

Thanks! Wayde.



From vfaronov at gmail.com  Sun Aug  6 18:11:03 2017
From: vfaronov at gmail.com (Vasiliy Faronov)
Date: Sun, 6 Aug 2017 21:11:03 +0300
Subject: Supporting the Forwarded header (RFC 7239)
Message-ID: <CALHHdhwaEXSftBiCUO06V6e6c0F=Wwpw+6NX5cyQ6U_F+PuKow@mail.gmail.com>

Hi nginx developers,

As you know, RFC 7239 defines a Forwarded header to replace the zoo of
X-Forwarded-* with a single extensible syntax.

There seems to be growing interest in deploying Forwarded. For
example, aiohttp, a popular Python library, recently started reading
Forwarded by default.

I'd like to see a $proxy_add_forwarded variable in nginx -- similar in
essence to $proxy_add_x_forwarded_for -- and I'd like to try and
contribute a patch.

Can you tell me if this sounds like a good idea to you, and if yes,
which of the following approaches you prefer? (I'm sorry for the wall
of text)


1. $proxy_add_forwarded blindly appends a "for=..." after a comma,
just like $proxy_add_x_forwarded_for does. (This cannot be emulated in
config because of ticket #1316; also, nginx's $remote_addr is not
quite in the format required by RFC 7239.)

The problem here: suppose an external user sends:

    Forwarded: for=injected;by="

If you just append to this, you get:

    Forwarded: for=injected;by=", for=real

This puts the burden on the receiving application to parse this
robustly and recover "for=real". They can't just split on comma
(because comma can occur inside a valid quoted-string), they need more
logic. It's doable, they just have to be aware of the problem and
actively mitigate it.


2. Parse any incoming Forwarded headers into an internal
representation, then re-serialize it with an added element.

This is obviously more expensive. But, if you think that supporting
Forwarded is a good idea, then eventually you want to support it in
ngx_http_realip_module and wherever else nginx looks at X-Forwarded-*.
Then you need to parse anyway.


3. Validate the syntax of any incoming Forwarded headers with a regex.
If they are valid, append to them. If they are invalid, replace them
with a single "for=unknown" (explicitly allowed by RFC 7239), and
append to that.


4. Do any of the above, but in a third-party module, where one could
experiment more freely.

I think requiring a third-party module to support Forwarded will just
lead to everybody sticking with X-Forwarded-*, or else trying to
emulate it in config with poor results. It's just not enough of a
feature on its own.

Then again, maybe RFC 7239 was a bad idea and everybody *should* just
stick to X-Forwarded-*.


Any thoughts on this would be appreciated. Thank you!


-- 
Vasiliy

From cornelis.bos at gmail.com  Mon Aug  7 07:44:28 2017
From: cornelis.bos at gmail.com (Kees Bos)
Date: Mon, 07 Aug 2017 09:44:28 +0200
Subject: [PATCH] Add proxy_protocol option to mail listener
In-Reply-To: <ca09cf40-7f60-e63a-8957-d14f0bb297d6@mcmaster.ca>
References: <ca09cf40-7f60-e63a-8957-d14f0bb297d6@mcmaster.ca>
Message-ID: <1502091868.14660.6.camel@gmail.com>

On za, 2017-08-05 at 20:44 -0400, Wayde Nie wrote:
> Hi Kees, Maxim,
> 
> I'm very interested in proxy_protocol enabled nginx mail proxy as
> well.
> Wondering if the feature might be more straightforward (initially?)
> if,
> when enabled on the listen line, it simply set the appropriate
> $proxy_protocol_* variables, similar to the http servers?
> 
> I was hoping that I could then include it as an http get param in the
> url for my auth_http directive for the auth script to do a dns
> blacklist
> lookup and/or logging (and possibly some other anti spambot efforts).
> 
> I'm working on it with your patch as a starting point, and I can get
> it
> to compile cleanly.??Nginx keeps working as expected when I don't set
> proxy_protocol on the listen directive, and when I do set
> proxy_protocol
> on the listen directive I see the correct ip and port picked up and
> logged in the error.log, however, then nginx stops sending the smtp
> greeting... My mail client connects to my loadbalancer, the lb
> connects
> to nginx:587 sending the PROXY line, nginx parses and logs the PROXY
> fields,??then the client times out waiting for any return traffic
> from
> nginx... I know it's something I'm doing :-)
> 
> I'm happy to keep poking away at it, but curious mostly, if you think
> the approach is sound? (ie. use $proxy_protocol_addr, set by
> proxy_protocol directive and pass in to auth_http script in auth url
> as
> a get param?) and if an initial patch that starts by just setting
> $proxy_protocol_* variables would be a useful first step in this type
> of
> functionality?
> 
> Thanks! Wayde.

Just to get the picture right (it looks to me that your downstream smtp
server expects the proxy protocol), what's the exact flow you're trying
to accomplish?


From ru at nginx.com  Mon Aug  7 09:06:48 2017
From: ru at nginx.com (Ruslan Ermilov)
Date: Mon, 07 Aug 2017 09:06:48 +0000
Subject: [nginx] Upstream: copy peer data in shared memory.
Message-ID: <hg.ac120e797d28.1502096808.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/ac120e797d28
branches:  
changeset: 7080:ac120e797d28
user:      Ruslan Ermilov <ru at nginx.com>
date:      Fri Aug 04 17:03:10 2017 +0300
description:
Upstream: copy peer data in shared memory.

This, in addition to 1eb753aa8e5e, fixes "upstream zone" on Windows.

diffstat:

 src/http/modules/ngx_http_upstream_zone_module.c |  77 +++++++++++++++++++++--
 src/stream/ngx_stream_upstream_zone_module.c     |  77 +++++++++++++++++++++--
 2 files changed, 138 insertions(+), 16 deletions(-)

diffs (226 lines):

diff -r 7564a919d333 -r ac120e797d28 src/http/modules/ngx_http_upstream_zone_module.c
--- a/src/http/modules/ngx_http_upstream_zone_module.c	Fri Aug 04 08:01:55 2017 +0300
+++ b/src/http/modules/ngx_http_upstream_zone_module.c	Fri Aug 04 17:03:10 2017 +0300
@@ -16,6 +16,8 @@ static ngx_int_t ngx_http_upstream_init_
     void *data);
 static ngx_http_upstream_rr_peers_t *ngx_http_upstream_zone_copy_peers(
     ngx_slab_pool_t *shpool, ngx_http_upstream_srv_conf_t *uscf);
+static ngx_http_upstream_rr_peer_t *ngx_http_upstream_zone_copy_peer(
+    ngx_http_upstream_rr_peers_t *peers, ngx_http_upstream_rr_peer_t *src);
 
 
 static ngx_command_t  ngx_http_upstream_zone_commands[] = {
@@ -215,14 +217,11 @@ ngx_http_upstream_zone_copy_peers(ngx_sl
 
     for (peerp = &peers->peer; *peerp; peerp = &peer->next) {
         /* pool is unlocked */
-        peer = ngx_slab_calloc_locked(shpool,
-                                      sizeof(ngx_http_upstream_rr_peer_t));
+        peer = ngx_http_upstream_zone_copy_peer(peers, *peerp);
         if (peer == NULL) {
             return NULL;
         }
 
-        ngx_memcpy(peer, *peerp, sizeof(ngx_http_upstream_rr_peer_t));
-
         *peerp = peer;
     }
 
@@ -243,14 +242,11 @@ ngx_http_upstream_zone_copy_peers(ngx_sl
 
     for (peerp = &backup->peer; *peerp; peerp = &peer->next) {
         /* pool is unlocked */
-        peer = ngx_slab_calloc_locked(shpool,
-                                      sizeof(ngx_http_upstream_rr_peer_t));
+        peer = ngx_http_upstream_zone_copy_peer(backup, *peerp);
         if (peer == NULL) {
             return NULL;
         }
 
-        ngx_memcpy(peer, *peerp, sizeof(ngx_http_upstream_rr_peer_t));
-
         *peerp = peer;
     }
 
@@ -262,3 +258,68 @@ done:
 
     return peers;
 }
+
+
+static ngx_http_upstream_rr_peer_t *
+ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers,
+    ngx_http_upstream_rr_peer_t *src)
+{
+    ngx_slab_pool_t              *pool;
+    ngx_http_upstream_rr_peer_t  *dst;
+
+    pool = peers->shpool;
+
+    dst = ngx_slab_calloc_locked(pool, sizeof(ngx_http_upstream_rr_peer_t));
+    if (dst == NULL) {
+        return NULL;
+    }
+
+    if (src) {
+        ngx_memcpy(dst, src, sizeof(ngx_http_upstream_rr_peer_t));
+        dst->sockaddr = NULL;
+        dst->name.data = NULL;
+        dst->server.data = NULL;
+    }
+
+    dst->sockaddr = ngx_slab_calloc_locked(pool, NGX_SOCKADDRLEN);
+    if (dst->sockaddr == NULL) {
+        goto failed;
+    }
+
+    dst->name.data = ngx_slab_calloc_locked(pool, NGX_SOCKADDR_STRLEN);
+    if (dst->name.data == NULL) {
+        goto failed;
+    }
+
+    if (src) {
+        ngx_memcpy(dst->sockaddr, src->sockaddr, src->socklen);
+        ngx_memcpy(dst->name.data, src->name.data, src->name.len);
+
+        dst->server.data = ngx_slab_alloc_locked(pool, src->server.len);
+        if (dst->server.data == NULL) {
+            goto failed;
+        }
+
+        ngx_memcpy(dst->server.data, src->server.data, src->server.len);
+    }
+
+    return dst;
+
+failed:
+
+    if (dst->server.data) {
+        ngx_slab_free_locked(pool, dst->server.data);
+    }
+
+    if (dst->name.data) {
+        ngx_slab_free_locked(pool, dst->name.data);
+    }
+
+    if (dst->sockaddr) {
+        ngx_slab_free_locked(pool, dst->sockaddr);
+    }
+
+    ngx_slab_free_locked(pool, dst);
+
+    return NULL;
+}
diff -r 7564a919d333 -r ac120e797d28 src/stream/ngx_stream_upstream_zone_module.c
--- a/src/stream/ngx_stream_upstream_zone_module.c	Fri Aug 04 08:01:55 2017 +0300
+++ b/src/stream/ngx_stream_upstream_zone_module.c	Fri Aug 04 17:03:10 2017 +0300
@@ -16,6 +16,8 @@ static ngx_int_t ngx_stream_upstream_ini
     void *data);
 static ngx_stream_upstream_rr_peers_t *ngx_stream_upstream_zone_copy_peers(
     ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf);
+static ngx_stream_upstream_rr_peer_t *ngx_stream_upstream_zone_copy_peer(
+    ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *src);
 
 
 static ngx_command_t  ngx_stream_upstream_zone_commands[] = {
@@ -212,14 +214,11 @@ ngx_stream_upstream_zone_copy_peers(ngx_
 
     for (peerp = &peers->peer; *peerp; peerp = &peer->next) {
         /* pool is unlocked */
-        peer = ngx_slab_calloc_locked(shpool,
-                                      sizeof(ngx_stream_upstream_rr_peer_t));
+        peer = ngx_stream_upstream_zone_copy_peer(peers, *peerp);
         if (peer == NULL) {
             return NULL;
         }
 
-        ngx_memcpy(peer, *peerp, sizeof(ngx_stream_upstream_rr_peer_t));
-
         *peerp = peer;
     }
 
@@ -240,14 +239,11 @@ ngx_stream_upstream_zone_copy_peers(ngx_
 
     for (peerp = &backup->peer; *peerp; peerp = &peer->next) {
         /* pool is unlocked */
-        peer = ngx_slab_calloc_locked(shpool,
-                                      sizeof(ngx_stream_upstream_rr_peer_t));
+        peer = ngx_stream_upstream_zone_copy_peer(backup, *peerp);
         if (peer == NULL) {
             return NULL;
         }
 
-        ngx_memcpy(peer, *peerp, sizeof(ngx_stream_upstream_rr_peer_t));
-
         *peerp = peer;
     }
 
@@ -259,3 +255,68 @@ done:
 
     return peers;
 }
+
+
+static ngx_stream_upstream_rr_peer_t *
+ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers,
+    ngx_stream_upstream_rr_peer_t *src)
+{
+    ngx_slab_pool_t                *pool;
+    ngx_stream_upstream_rr_peer_t  *dst;
+
+    pool = peers->shpool;
+
+    dst = ngx_slab_calloc_locked(pool, sizeof(ngx_stream_upstream_rr_peer_t));
+    if (dst == NULL) {
+        return NULL;
+    }
+
+    if (src) {
+        ngx_memcpy(dst, src, sizeof(ngx_stream_upstream_rr_peer_t));
+        dst->sockaddr = NULL;
+        dst->name.data = NULL;
+        dst->server.data = NULL;
+    }
+
+    dst->sockaddr = ngx_slab_calloc_locked(pool, NGX_SOCKADDRLEN);
+    if (dst->sockaddr == NULL) {
+        goto failed;
+    }
+
+    dst->name.data = ngx_slab_calloc_locked(pool, NGX_SOCKADDR_STRLEN);
+    if (dst->name.data == NULL) {
+        goto failed;
+    }
+
+    if (src) {
+        ngx_memcpy(dst->sockaddr, src->sockaddr, src->socklen);
+        ngx_memcpy(dst->name.data, src->name.data, src->name.len);
+
+        dst->server.data = ngx_slab_alloc_locked(pool, src->server.len);
+        if (dst->server.data == NULL) {
+            goto failed;
+        }
+
+        ngx_memcpy(dst->server.data, src->server.data, src->server.len);
+    }
+
+    return dst;
+
+failed:
+
+    if (dst->server.data) {
+        ngx_slab_free_locked(pool, dst->server.data);
+    }
+
+    if (dst->name.data) {
+        ngx_slab_free_locked(pool, dst->name.data);
+    }
+
+    if (dst->sockaddr) {
+        ngx_slab_free_locked(pool, dst->sockaddr);
+    }
+
+    ngx_slab_free_locked(pool, dst);
+
+    return NULL;
+}

From eran.kornblau at kaltura.com  Mon Aug  7 10:36:19 2017
From: eran.kornblau at kaltura.com (Eran Kornblau)
Date: Mon, 7 Aug 2017 10:36:19 +0000
Subject: Add support for buffering is scripted logs
Message-ID: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>

Hi all,

The attached patch adds support for log buffering when using variables in the access log file name.

The use case is this - we use nginx to receive analytics beacons and write them to the access log.
We'd like to have a log file per hour that contains the logs of the specific hour. If we use some external 
script to perform log rotate, we cannot avoid log lines slipping between adjacent files.
It's important for us to have the log lines partitioning accurate, in case we need to reindex a specific 
hour to the database.
Scripted logs seem perfect for this, but we don't want to give up on log compression, since the 
number of events can be high.

The patch relies on the open file cache to keep the context (buffer + flush event) of each file.
In order to do that, I added the ability to register callbacks in open file cache for these events:
1. init - new cached file object created
2. flush - the file handle of a cached file is closed
3. free - the cached file is destroyed
The context is allocated right after the ngx_cached_open_file_t struct, this was done in order to avoid
increasing the size of the ngx_cached_open_file_t struct (compared to the alternative of adding some
opaque pointer on this struct).
Therefore, there are no memory implications for "regular" (log = 0) open file caches, the overhead is
only an extra 'if' on the creation / deletion of the cached file object (no impact at all on cache hit,
which is probably the most performance sensitive) 

Btw, on a similar subject - can anyone explain the purpose of checking the existence of the root dir
in scripted logs? Only explanation I could think of is to provide some security protection in case $uri
is used in the script, but that sounds like a very specific use case to me... 
Anyway, if that is indeed the case, maybe it makes sense to add conf directive to enable/disable 
this behavior?

Thank you!

Eran

-------------- next part --------------
A non-text attachment was scrubbed...
Name: buffered_script_logs.patch
Type: application/octet-stream
Size: 19571 bytes
Desc: buffered_script_logs.patch
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170807/5602186a/attachment-0001.obj>

From eran.kornblau at kaltura.com  Mon Aug  7 11:08:49 2017
From: eran.kornblau at kaltura.com (Eran Kornblau)
Date: Mon, 7 Aug 2017 11:08:49 +0000
Subject: Add support for buffering in scripted logs
Message-ID: <AM4PR0401MB1731BFCCE414FD22A7FCF916F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>

Sorry for spamming... forgot to write - something that I know is missing in this patch is support for 'reopen files' (USR1)
In order to handle that, I probably need to add an rbtree on cycle->open_files (the O(n) search that is currently performed 
in ngx_conf_open_file doesn't seem appropriate for runtime)
If the general concept in the below patch is approved, I will happily submit a patch for this as well.

Thanks

Eran

-----Original Message-----
From: Eran Kornblau 
Sent: Monday, August 7, 2017 1:36 PM
To: nginx-devel at nginx.org
Subject: Add support for buffering is scripted logs

Hi all,

The attached patch adds support for log buffering when using variables in the access log file name.

The use case is this - we use nginx to receive analytics beacons and write them to the access log.
We'd like to have a log file per hour that contains the logs of the specific hour. If we use some external script to perform log rotate, we cannot avoid log lines slipping between adjacent files.
It's important for us to have the log lines partitioning accurate, in case we need to reindex a specific hour to the database.
Scripted logs seem perfect for this, but we don't want to give up on log compression, since the number of events can be high.

The patch relies on the open file cache to keep the context (buffer + flush event) of each file.
In order to do that, I added the ability to register callbacks in open file cache for these events:
1. init - new cached file object created 2. flush - the file handle of a cached file is closed 3. free - the cached file is destroyed The context is allocated right after the ngx_cached_open_file_t struct, this was done in order to avoid increasing the size of the ngx_cached_open_file_t struct (compared to the alternative of adding some opaque pointer on this struct).
Therefore, there are no memory implications for "regular" (log = 0) open file caches, the overhead is only an extra 'if' on the creation / deletion of the cached file object (no impact at all on cache hit, which is probably the most performance sensitive) 

Btw, on a similar subject - can anyone explain the purpose of checking the existence of the root dir in scripted logs? Only explanation I could think of is to provide some security protection in case $uri is used in the script, but that sounds like a very specific use case to me... 
Anyway, if that is indeed the case, maybe it makes sense to add conf directive to enable/disable this behavior?

Thank you!

Eran


From niew at mcmaster.ca  Mon Aug  7 11:49:52 2017
From: niew at mcmaster.ca (Wayde Nie)
Date: Mon, 7 Aug 2017 07:49:52 -0400
Subject: [PATCH] Add proxy_protocol option to mail listener
In-Reply-To: <1502091868.14660.6.camel@gmail.com>
References: <ca09cf40-7f60-e63a-8957-d14f0bb297d6@mcmaster.ca>
 <1502091868.14660.6.camel@gmail.com>
Message-ID: <8f8e9d67-191a-92b1-990d-bdde41a5fe68@mcmaster.ca>

On 08/07/2017 03:44 AM, Kees Bos wrote:, and when I do set
>> proxy_protocol
>> on the listen directive I see the correct ip and port picked up and
>> logged in the error.log, however, then nginx stops sending the smtp
>> greeting... My mail client connects to my loadbalancer, the lb
>> connects
>> to nginx:587 sending the PROXY line, nginx parses and logs the PROXY
>> fields,  then the client times out waiting for any return traffic
>> from
>> nginx... I know it's something I'm doing :-)
>>
>> I'm happy to keep poking away at it, but curious mostly, if you think
>> the approach is sound? (ie. use $proxy_protocol_addr, set by
>> proxy_protocol directive and pass in to auth_http script in auth url
>> as
>> a get param?) and if an initial patch that starts by just setting
>> $proxy_protocol_* variables would be a useful first step in this type
>> of
>> functionality?
>>> Just to get the picture right (it looks to me that your downstream smtp
>>> server expects the proxy protocol), what's the exact flow you're trying
>>> to accomplish?

 Hi Kees,

Thanks for looking!

In my use case I have an external hardware loadbalancer that is
receiving end user connections on a VIP, pre-pending proxy_protocol
header and loadbalancing them to small pool of nginx servers running as
the mail proxies. Nginx is parsing the proxy_protocol header and (I
hope) proxying to my upstream smtp server without passing the
proxy_protocol header, which my upstream smtp server doesn't support (as
currently implemented).

Flow like:

    1) Client makes connection to [external-LB-VIP:587] for email submission
    2) [external-LB-VIP:587] --> injects proxy_protocol header --> load
    balances to set of nginx services via TCP service pool (ie: lb
    straight tcp, no application level inspection by loadbalancer, other
    than prepending proxy_protocol header)
    3) Nginx parses proxy_protocol header, logs client ip and passes
    client IP into auth_http script, along with username and password
    for authn/authz
    4) on successful return response from auth_http; nginx proxies mail
    submission to upstream smtp server without proxy_protocol header.


So, if possible, I'd like nginx to get the client ip passed to it from
the external hardware load balancer, log it and then use it in the
auth_http script, but otherwise not pass it on to the upstream smtp
server...

Is this doable?

Thanks,Wayde. 
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170807/97cb36d0/attachment.html>

From mdounin at mdounin.ru  Mon Aug  7 13:20:49 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 7 Aug 2017 16:20:49 +0300
Subject: [PATCH] Allows hyphens in variable names so can access hyphenated
 query vars
In-Reply-To: <08008b0d1f4ffbd017d9.1501673411@ma.h1.thisaintnews.com>
References: <08008b0d1f4ffbd017d9.1501673411@ma.h1.thisaintnews.com>
Message-ID: <20170807132049.GZ93611@mdounin.ru>

Hello!

On Wed, Aug 02, 2017 at 11:30:11AM +0000, Mark Ellis via nginx-devel wrote:

> # HG changeset patch
> # User Mark Ellis <m at rkellis.com>
> # Date 1501670303 -3600
> #      Wed Aug 02 11:38:23 2017 +0100
> # Node ID 08008b0d1f4ffbd017d9f50142bdd46d7c4a41d2
> # Parent  1eb753aa8e5e9ee5059ebb544194c8919b674873
> Allows hyphens in variable names so can access hyphenated query vars
> 
> This patch allows hyphens in variable names so that you can access query string
> parameters with hyphens
> 
> Before if you had a query string param such as `?foo-bar=123` you could not
> access it with the variable `$foo-bar` as variables could not contain hyphens.
> 
> diff -r 1eb753aa8e5e -r 08008b0d1f4f src/http/ngx_http_script.c
> --- a/src/http/ngx_http_script.c	Tue Aug 01 19:12:10 2017 +0300
> +++ b/src/http/ngx_http_script.c	Wed Aug 02 11:38:23 2017 +0100
> @@ -403,6 +403,7 @@
>                  if ((ch >= 'A' && ch <= 'Z')
>                      || (ch >= 'a' && ch <= 'z')
>                      || (ch >= '0' && ch <= '9')
> +                    || ch == '-'
>                      || ch == '_')
>                  {
>                      continue;

No, thanks.  Such a change will break a lot of configs, not to 
mention it is counter-intuitive and contradicts how things usually 
work in other programming languages.

-- 
Maxim Dounin
http://nginx.org/

From cornelis.bos at gmail.com  Mon Aug  7 13:23:12 2017
From: cornelis.bos at gmail.com (Kees Bos)
Date: Mon, 07 Aug 2017 15:23:12 +0200
Subject: [PATCH] Add proxy_protocol option to mail listener
Message-ID: <1502112192.14660.18.camel@gmail.com>

# HG changeset patch
# User Kees Bos <cornelis.bos at gmail.com>
# Date 1500565189 0
#??????Thu Jul 20 15:39:49 2017 +0000
# Node ID 327f18e079b175b14277a23e75715f5feee34d69
# Parent??863b862534d7ac0dbf8babf68b824de6fb0d6ef4
Add proxy_protocol option to mail listener

Add support for the mail handlers. This enables the use of an upstream
loadbalancer/proxy that connects with the proxy protocol. Examples of
this are haproxy or a nginx stream handler that uses then proxy protocol
in client conections.

The proxy protocol source ip address will we exposed to the auth
handler as 'Proxy-Protocol-IP'.

If the sender ip address matches one or more "set_real_ip_from" directives,
the source ip address as specified in the in the proxy protocol will be
used as 'Client-IP' in the authentication call and as address in the
XCLIENT call.

Example config:
mail {
????server_name mail.example.com;
????auth_http???localhost:9000/;

????server {
????????listen 143 proxy_protocol;
????????protocol imap;
????}

????server {
????????listen 25 proxy_protocol;
????????protocol smtp;
????????set_real_ip_from 127.0.0.0/8;
????????set_real_ip_from ::/128;
????}
}

In the imap config, the source address given in the proxy protocol will
never be used as Client-IP.

In the smtp config, the source address given in the proxy protocol will
only be used as XCLIENT address when the sender address matches the
"set_real_ip_from" settings (in this case only loopback address).

diff -r 863b862534d7 -r 327f18e079b1 auto/modules
--- a/auto/modules	Wed Jul 19 21:39:40 2017 +0800
+++ b/auto/modules	Thu Jul 20 15:39:49 2017 +0000
@@ -954,6 +954,12 @@
?????ngx_module_srcs=src/mail/ngx_mail_proxy_module.c
?
?????. auto/module
+
+????ngx_module_name=ngx_mail_realip_module
+????ngx_module_deps=
+????ngx_module_srcs=src/mail/ngx_mail_realip_module.c
+
+????. auto/module
?fi
?
?
diff -r 863b862534d7 -r 327f18e079b1 src/mail/ngx_mail.c
--- a/src/mail/ngx_mail.c	Wed Jul 19 21:39:40 2017 +0800
+++ b/src/mail/ngx_mail.c	Thu Jul 20 15:39:49 2017 +0000
@@ -408,6 +408,7 @@
?#if (NGX_MAIL_SSL)
?????????addrs[i].conf.ssl = addr[i].opt.ssl;
?#endif
+????????addrs[i].conf.proxy_protocol = addr[i].opt.proxy_protocol;
?
?????????len = ngx_sock_ntop(&addr[i].opt.sockaddr.sockaddr, addr[i].opt.socklen,
?????????????????????????????buf, NGX_SOCKADDR_STRLEN, 1);
@@ -457,6 +458,7 @@
?#if (NGX_MAIL_SSL)
?????????addrs6[i].conf.ssl = addr[i].opt.ssl;
?#endif
+????????addrs6[i].conf.proxy_protocol = addr[i].opt.proxy_protocol;
?
?????????len = ngx_sock_ntop(&addr[i].opt.sockaddr.sockaddr, addr[i].opt.socklen,
?????????????????????????????buf, NGX_SOCKADDR_STRLEN, 1);
diff -r 863b862534d7 -r 327f18e079b1 src/mail/ngx_mail.h
--- a/src/mail/ngx_mail.h	Wed Jul 19 21:39:40 2017 +0800
+++ b/src/mail/ngx_mail.h	Thu Jul 20 15:39:49 2017 +0000
@@ -40,6 +40,7 @@
?????unsigned????????????????ipv6only:1;
?#endif
?????unsigned????????????????so_keepalive:2;
+????unsigned????????????????proxy_protocol:1;
?#if (NGX_HAVE_KEEPALIVE_TUNABLE)
?????int?????????????????????tcp_keepidle;
?????int?????????????????????tcp_keepintvl;
@@ -54,7 +55,8 @@
?typedef struct {
?????ngx_mail_conf_ctx_t????*ctx;
?????ngx_str_t???????????????addr_text;
-????ngx_uint_t??????????????ssl;????/* unsigned???ssl:1; */
+????unsigned????????????????ssl:1;
+????unsigned????????????????proxy_protocol:1;
?} ngx_mail_addr_conf_t;
?
?typedef struct {
@@ -204,6 +206,8 @@
?????unsigned????????????????esmtp:1;
?????unsigned????????????????auth_method:3;
?????unsigned????????????????auth_wait:1;
+????unsigned????????????????ssl:1;
+????unsigned????????????????proxy_protocol:1;
?
?????ngx_str_t???????????????login;
?????ngx_str_t???????????????passwd;
@@ -371,6 +375,8 @@
?#endif
?
?
+ngx_int_t ngx_mail_realip_handler(ngx_mail_session_t *s);
+
?void ngx_mail_init_connection(ngx_connection_t *c);
?
?ngx_int_t ngx_mail_salt(ngx_mail_session_t *s, ngx_connection_t *c,
diff -r 863b862534d7 -r 327f18e079b1 src/mail/ngx_mail_auth_http_module.c
--- a/src/mail/ngx_mail_auth_http_module.c	Wed Jul 19 21:39:40 2017 +0800
+++ b/src/mail/ngx_mail_auth_http_module.c	Thu Jul 20 15:39:49 2017 +0000
@@ -1134,7 +1134,7 @@
?{
?????size_t?????????????????????len;
?????ngx_buf_t?????????????????*b;
-????ngx_str_t??????????????????login, passwd;
+????ngx_str_t?????????????????*client_addr, login, passwd;
?#if (NGX_MAIL_SSL)
?????ngx_str_t??????????????????verify, subject, issuer, serial, fingerprint,
????????????????????????????????raw_cert, cert;
@@ -1209,6 +1209,12 @@
?
?????cscf = ngx_mail_get_module_srv_conf(s, ngx_mail_core_module);
?
+????if (ngx_mail_realip_handler(s) == NGX_OK) {
+????????client_addr = &s->connection->proxy_protocol_addr;
+????} else {
+????????client_addr = &s->connection->addr_text;
+????}
+
?????len = sizeof("GET ") - 1 + ahcf->uri.len + sizeof(" HTTP/1.0" CRLF) - 1
???????????+ sizeof("Host: ") - 1 + ahcf->host_header.len + sizeof(CRLF) - 1
???????????+ sizeof("Auth-Method: ") - 1
@@ -1221,8 +1227,7 @@
?????????????????+ sizeof(CRLF) - 1
???????????+ sizeof("Auth-Login-Attempt: ") - 1 + NGX_INT_T_LEN
?????????????????+ sizeof(CRLF) - 1
-??????????+ sizeof("Client-IP: ") - 1 + s->connection->addr_text.len
-????????????????+ sizeof(CRLF) - 1
+??????????+ sizeof("Client-IP: ") - 1 + client_addr->len + sizeof(CRLF) - 1
???????????+ sizeof("Client-Host: ") - 1 + s->host.len + sizeof(CRLF) - 1
???????????+ sizeof("Auth-SMTP-Helo: ") - 1 + s->smtp_helo.len + sizeof(CRLF) - 1
???????????+ sizeof("Auth-SMTP-From: ") - 1 + s->smtp_from.len + sizeof(CRLF) - 1
@@ -1240,6 +1245,11 @@
???????????+ ahcf->header.len
???????????+ sizeof(CRLF) - 1;
?
+????if (s->connection->proxy_protocol_addr.len) {
+????????len += sizeof("Proxy-Protocol-IP: ") - 1 +
+???????????????s->connection->proxy_protocol_addr.len + sizeof(CRLF) - 1;
+????}
+
?????b = ngx_create_temp_buf(pool, len);
?????if (b == NULL) {
?????????return NULL;
@@ -1287,8 +1297,7 @@
???????????????????????????s->login_attempt);
?
?????b->last = ngx_cpymem(b->last, "Client-IP: ", sizeof("Client-IP: ") - 1);
-????b->last = ngx_copy(b->last, s->connection->addr_text.data,
-???????????????????????s->connection->addr_text.len);
+????b->last = ngx_copy(b->last, client_addr->data, client_addr->len);
?????*b->last++ = CR; *b->last++ = LF;
?
?????if (s->host.len) {
@@ -1298,6 +1307,14 @@
?????????*b->last++ = CR; *b->last++ = LF;
?????}
?
+????if (s->connection->proxy_protocol_addr.len) {
+????????b->last = ngx_cpymem(b->last, "Proxy-Protocol-IP: ",
+?????????????????????????????sizeof("Proxy-Protocol-IP: ") - 1);
+????????b->last = ngx_copy(b->last, s->connection->proxy_protocol_addr.data,
+???????????????????????????s->connection->proxy_protocol_addr.len);
+????????*b->last++ = CR; *b->last++ = LF;
+????}
+
?????if (s->auth_method == NGX_MAIL_AUTH_NONE) {
?
?????????/* HELO, MAIL FROM, and RCPT TO can't contain CRLF, no need to escape */
diff -r 863b862534d7 -r 327f18e079b1 src/mail/ngx_mail_core_module.c
--- a/src/mail/ngx_mail_core_module.c	Wed Jul 19 21:39:40 2017 +0800
+++ b/src/mail/ngx_mail_core_module.c	Thu Jul 20 15:39:49 2017 +0000
@@ -574,6 +574,11 @@
?#endif
?????????}
?
+????????if (ngx_strcmp(value[i].data, "proxy_protocol") == 0) {
+????????????ls->proxy_protocol = 1;
+????????????continue;
+????????}
+
?????????ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
????????????????????????????"the invalid \"%V\" parameter", &value[i]);
?????????return NGX_CONF_ERROR;
diff -r 863b862534d7 -r 327f18e079b1 src/mail/ngx_mail_handler.c
--- a/src/mail/ngx_mail_handler.c	Wed Jul 19 21:39:40 2017 +0800
+++ b/src/mail/ngx_mail_handler.c	Thu Jul 20 15:39:49 2017 +0000
@@ -12,6 +12,7 @@
?
?
?static void ngx_mail_init_session(ngx_connection_t *c);
+static void ngx_mail_proxy_protocol_handler(ngx_event_t *rev);
?
?#if (NGX_MAIL_SSL)
?static void ngx_mail_ssl_init_connection(ngx_ssl_t *ssl, ngx_connection_t *c);
@@ -143,6 +144,8 @@
?????ngx_log_error(NGX_LOG_INFO, c->log, 0, "*%uA client %*s connected to %V",
???????????????????c->number, len, text, s->addr_text);
?
+????s->proxy_protocol = addr_conf->proxy_protocol;
+
?????ctx = ngx_palloc(c->pool, sizeof(ngx_mail_log_ctx_t));
?????if (ctx == NULL) {
?????????ngx_mail_close_connection(c);
@@ -165,16 +168,11 @@
?
?????sslcf = ngx_mail_get_module_srv_conf(s, ngx_mail_ssl_module);
?
+????s->ssl = 0;
?????if (sslcf->enable) {
-????????c->log->action = "SSL handshaking";
-
-????????ngx_mail_ssl_init_connection(&sslcf->ssl, c);
-????????return;
-????}
-
-????if (addr_conf->ssl) {
-
-????????c->log->action = "SSL handshaking";
+????????s->ssl = 1;
+????} else if (addr_conf->ssl) {
+????????s->ssl = 1;
?
?????????if (sslcf->ssl.ctx == NULL) {
?????????????ngx_log_error(NGX_LOG_ERR, c->log, 0,
@@ -183,11 +181,97 @@
?????????????ngx_mail_close_connection(c);
?????????????return;
?????????}
+????}
?
+????}
+#endif
+
+????if (s->proxy_protocol) {
+????????c->log->action = "reading PROXY protocol";
+
+????????ngx_add_timer(c->read, cscf->timeout);
+????????c->read->handler = ngx_mail_proxy_protocol_handler;
+
+????????if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+????????????ngx_mail_close_connection(c);
+????????}
+????????return;
+????}
+
+
+#if (NGX_MAIL_SSL)
+????if (s->ssl) {
+????????ngx_mail_ssl_conf_t??*sslcf;
+????????c->log->action = "SSL handshaking";
+
+????????sslcf = ngx_mail_get_module_srv_conf(s, ngx_mail_ssl_module);
?????????ngx_mail_ssl_init_connection(&sslcf->ssl, c);
?????????return;
?????}
+# endif
?
+????ngx_mail_init_session(c);
+}
+
+
+void
+ngx_mail_proxy_protocol_handler(ngx_event_t *rev)
+{
+????u_char????????????????????*p, buf[NGX_PROXY_PROTOCOL_MAX_HEADER + 1],
+???????????????????????????????text[NGX_SOCKADDR_STRLEN];
+????size_t?????????????????????size, len;
+????ssize_t????????????????????n;
+????ngx_err_t??????????????????err;
+????ngx_connection_t??????????*c;
+????ngx_mail_session_t????????*s;
+????ngx_mail_log_ctx_t????????*ctx;
+
+????size = sizeof(buf);
+????c = rev->data;
+
+????n = recv(c->fd, (char *) buf, size, MSG_PEEK);
+????err = ngx_socket_errno;
+
+????if (n == -1) {
+????????ngx_connection_error(c, err, "recv() failed");
+????????ngx_mail_close_connection(c);
+????????return;
+????}
+
+????p = ngx_proxy_protocol_read(c, buf, buf + n);
+????if (p == NULL) {
+????????ngx_mail_close_connection(c);
+????????return;
+????}
+
+????size = p - buf;
+????if (c->recv(c, buf, size) != (ssize_t) size) {
+????????ngx_mail_close_connection(c);
+????????return;
+????}
+
+????s = c->data;
+????s->proxy_protocol = 0;
+
+????ctx = c->log->data;
+????ctx->client = &c->proxy_protocol_addr;
+
+????len = ngx_sock_ntop(c->sockaddr, c->socklen, text, NGX_SOCKADDR_STRLEN, 1);
+????ngx_log_error(NGX_LOG_INFO, c->log, 0,
+??????????????????"*%uA client %*s proxy-protocol %*s:%d",
+??????????????????c->number, len, text,
+??????????????????c->proxy_protocol_addr.len, c->proxy_protocol_addr.data,
+??????????????????c->proxy_protocol_port);
+
+#if (NGX_MAIL_SSL)
+????{
+????ngx_mail_ssl_conf_t??*sslcf;
+????if (s->ssl) {
+????????c->log->action = "SSL handshaking";
+????????sslcf = ngx_mail_get_module_srv_conf(s, ngx_mail_ssl_module);
+????????ngx_mail_ssl_init_connection(&sslcf->ssl, c);
+????????return;
+????}
?????}
?#endif
?
@@ -222,6 +306,21 @@
?????ngx_mail_session_t????????*s;
?????ngx_mail_core_srv_conf_t??*cscf;
?
+????s = c->data;
+????cscf = ngx_mail_get_module_srv_conf(s, ngx_mail_core_module);
+
+????if (s->proxy_protocol) {
+????????c->log->action = "reading PROXY protocol";
+
+????????ngx_add_timer(c->read, cscf->timeout);
+????????c->read->handler = ngx_mail_proxy_protocol_handler;
+
+????????if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+????????????ngx_mail_close_connection(c);
+????????}
+????????return;
+????}
+
?????if (ngx_ssl_create_connection(ssl, c, 0) != NGX_OK) {
?????????ngx_mail_close_connection(c);
?????????return;
@@ -229,10 +328,6 @@
?
?????if (ngx_ssl_handshake(c) == NGX_AGAIN) {
?
-????????s = c->data;
-
-????????cscf = ngx_mail_get_module_srv_conf(s, ngx_mail_core_module);
-
?????????ngx_add_timer(c->read, cscf->timeout);
?
?????????c->ssl->handler = ngx_mail_ssl_handshake_handler;
diff -r 863b862534d7 -r 327f18e079b1 src/mail/ngx_mail_proxy_module.c
--- a/src/mail/ngx_mail_proxy_module.c	Wed Jul 19 21:39:40 2017 +0800
+++ b/src/mail/ngx_mail_proxy_module.c	Thu Jul 20 15:39:49 2017 +0000
@@ -450,7 +450,7 @@
?{
?????u_char????????????????????*p;
?????ngx_int_t??????????????????rc;
-????ngx_str_t??????????????????line;
+????ngx_str_t?????????????????*client_addr, line;
?????ngx_buf_t?????????????????*b;
?????ngx_connection_t??????????*c;
?????ngx_mail_session_t????????*s;
@@ -523,11 +523,17 @@
?????????ngx_log_debug0(NGX_LOG_DEBUG_MAIL, rev->log, 0,
????????????????????????"mail proxy send xclient");
?
+????????if (ngx_mail_realip_handler(s) == NGX_OK) {
+????????????client_addr = &s->connection->proxy_protocol_addr;
+????????} else {
+????????????client_addr = &s->connection->addr_text;
+????????}
+
?????????s->connection->log->action = "sending XCLIENT to upstream";
?
?????????line.len = sizeof("XCLIENT ADDR= LOGIN= NAME="
???????????????????????????CRLF) - 1
-???????????????????+ s->connection->addr_text.len + s->login.len + s->host.len;
+???????????????????+ client_addr->len + s->login.len + s->host.len;
?
?#if (NGX_HAVE_INET6)
?????????if (s->connection->sockaddr->sa_family == AF_INET6) {
@@ -549,8 +555,7 @@
?????????}
?#endif
?
-????????p = ngx_copy(p, s->connection->addr_text.data,
-?????????????????????s->connection->addr_text.len);
+????????p = ngx_copy(p, client_addr->data, client_addr->len);
?
?????????if (s->login.len) {
?????????????p = ngx_cpymem(p, " LOGIN=", sizeof(" LOGIN=") - 1);
diff -r 863b862534d7 -r 327f18e079b1 src/mail/ngx_mail_realip_module.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/mail/ngx_mail_realip_module.c	Thu Jul 20 15:39:49 2017 +0000
@@ -0,0 +1,232 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_mail.h>
+
+
+typedef struct {
+????ngx_array_t???????*from;?????/* array of ngx_cidr_t */
+} ngx_mail_realip_srv_conf_t;
+
+
+typedef struct {
+????struct sockaddr???*sockaddr;
+????socklen_t??????????socklen;
+????ngx_str_t??????????addr_text;
+} ngx_mail_realip_ctx_t;
+
+
+static char *ngx_mail_realip_from(ngx_conf_t *cf, ngx_command_t *cmd,
+????void *conf);
+static void *ngx_mail_realip_create_srv_conf(ngx_conf_t *cf);
+static char *ngx_mail_realip_merge_srv_conf(ngx_conf_t *cf, void *parent,
+????void *child);
+
+
+static ngx_command_t??ngx_mail_realip_commands[] = {
+
+????{ ngx_string("set_realip_from"),
+??????NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_TAKE1,
+??????ngx_mail_realip_from,
+??????NGX_MAIL_SRV_CONF_OFFSET,
+??????0,
+??????NULL },
+
+??????ngx_null_command
+};
+
+
+static ngx_mail_module_t??ngx_mail_realip_module_ctx = {
+????NULL,??????????????????????????????????/* protocol */
+
+????NULL,??????????????????????????????????/* create main configuration */
+????NULL,??????????????????????????????????/* init main configuration */
+
+????ngx_mail_realip_create_srv_conf,???????/* create server configuration */
+????ngx_mail_realip_merge_srv_conf?????????/* merge server configuration */
+};
+
+
+ngx_module_t??ngx_mail_realip_module = {
+????NGX_MODULE_V1,
+????&ngx_mail_realip_module_ctx,???????????/* module context */
+????ngx_mail_realip_commands,??????????????/* module directives */
+????NGX_MAIL_MODULE,???????????????????????/* module type */
+????NULL,??????????????????????????????????/* init master */
+????NULL,??????????????????????????????????/* init module */
+????NULL,??????????????????????????????????/* init process */
+????NULL,??????????????????????????????????/* init thread */
+????NULL,??????????????????????????????????/* exit thread */
+????NULL,??????????????????????????????????/* exit process */
+????NULL,??????????????????????????????????/* exit master */
+????NGX_MODULE_V1_PADDING
+};
+
+
+ngx_int_t
+ngx_mail_realip_handler(ngx_mail_session_t *s)
+{
+????ngx_connection_t??????????????*c;
+????ngx_mail_realip_srv_conf_t????*rscf;
+
+????rscf = ngx_mail_get_module_srv_conf(s, ngx_mail_realip_module);
+
+????if (rscf->from == NULL) {
+????????return NGX_DECLINED;
+????}
+
+????c = s->connection;
+
+????if (c->proxy_protocol_addr.len == 0) {
+????????return NGX_DECLINED;
+????}
+
+????if (ngx_cidr_match(c->sockaddr, rscf->from) != NGX_OK) {
+????????return NGX_DECLINED;
+????}
+
+????return NGX_OK;
+}
+
+
+static char *
+ngx_mail_realip_from(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+????ngx_mail_realip_srv_conf_t *rscf = conf;
+
+????ngx_int_t?????????????rc;
+????ngx_str_t????????????*value;
+????ngx_url_t?????????????u;
+????ngx_cidr_t????????????c, *cidr;
+????ngx_uint_t????????????i;
+????struct sockaddr_in???*sin;
+#if (NGX_HAVE_INET6)
+????struct sockaddr_in6??*sin6;
+#endif
+
+????value = cf->args->elts;
+
+????if (rscf->from == NULL) {
+????????rscf->from = ngx_array_create(cf->pool, 2,
+??????????????????????????????????????sizeof(ngx_cidr_t));
+????????if (rscf->from == NULL) {
+????????????return NGX_CONF_ERROR;
+????????}
+????}
+
+#if (NGX_HAVE_UNIX_DOMAIN)
+
+????if (ngx_strcmp(value[1].data, "unix:") == 0) {
+????????cidr = ngx_array_push(rscf->from);
+????????if (cidr == NULL) {
+????????????return NGX_CONF_ERROR;
+????????}
+
+????????cidr->family = AF_UNIX;
+????????return NGX_CONF_OK;
+????}
+
+#endif
+
+????rc = ngx_ptocidr(&value[1], &c);
+
+????if (rc != NGX_ERROR) {
+????????if (rc == NGX_DONE) {
+????????????ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
+???????????????????????????????"low address bits of %V are meaningless",
+???????????????????????????????&value[1]);
+????????}
+
+????????cidr = ngx_array_push(rscf->from);
+????????if (cidr == NULL) {
+????????????return NGX_CONF_ERROR;
+????????}
+
+????????*cidr = c;
+
+????????return NGX_CONF_OK;
+????}
+
+????ngx_memzero(&u, sizeof(ngx_url_t));
+????u.host = value[1];
+
+????if (ngx_inet_resolve_host(cf->pool, &u) != NGX_OK) {
+????????if (u.err) {
+????????????ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+???????????????????????????????"%s in set_real_ip_from \"%V\"",
+???????????????????????????????u.err, &u.host);
+????????}
+
+????????return NGX_CONF_ERROR;
+????}
+
+????cidr = ngx_array_push_n(rscf->from, u.naddrs);
+????if (cidr == NULL) {
+????????return NGX_CONF_ERROR;
+????}
+
+????ngx_memzero(cidr, u.naddrs * sizeof(ngx_cidr_t));
+
+????for (i = 0; i < u.naddrs; i++) {
+????????cidr[i].family = u.addrs[i].sockaddr->sa_family;
+
+????????switch (cidr[i].family) {
+
+#if (NGX_HAVE_INET6)
+????????case AF_INET6:
+????????????sin6 = (struct sockaddr_in6 *) u.addrs[i].sockaddr;
+????????????cidr[i].u.in6.addr = sin6->sin6_addr;
+????????????ngx_memset(cidr[i].u.in6.mask.s6_addr, 0xff, 16);
+????????????break;
+#endif
+
+????????default: /* AF_INET */
+????????????sin = (struct sockaddr_in *) u.addrs[i].sockaddr;
+????????????cidr[i].u.in.addr = sin->sin_addr.s_addr;
+????????????cidr[i].u.in.mask = 0xffffffff;
+????????????break;
+????????}
+????}
+
+????return NGX_CONF_OK;
+}
+
+
+static void *
+ngx_mail_realip_create_srv_conf(ngx_conf_t *cf)
+{
+????ngx_mail_realip_srv_conf_t??*conf;
+
+????conf = ngx_pcalloc(cf->pool, sizeof(ngx_mail_realip_srv_conf_t));
+????if (conf == NULL) {
+????????return NULL;
+????}
+
+????/*
+?????* set by ngx_pcalloc():
+?????*
+?????*?????conf->from = NULL;
+?????*/
+
+????return conf;
+}
+
+
+static char *
+ngx_mail_realip_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
+{
+????ngx_mail_realip_srv_conf_t *prev = parent;
+????ngx_mail_realip_srv_conf_t *conf = child;
+
+????if (conf->from == NULL) {
+????????conf->from = prev->from;
+????}
+
+????return NGX_CONF_OK;
+}

From cornelis.bos at gmail.com  Mon Aug  7 13:25:24 2017
From: cornelis.bos at gmail.com (Kees Bos)
Date: Mon, 07 Aug 2017 15:25:24 +0200
Subject: [PATCH] Add proxy_protocol option to mail listener
In-Reply-To: <8f8e9d67-191a-92b1-990d-bdde41a5fe68@mcmaster.ca>
References: <ca09cf40-7f60-e63a-8957-d14f0bb297d6@mcmaster.ca>
 <1502091868.14660.6.camel@gmail.com>
 <8f8e9d67-191a-92b1-990d-bdde41a5fe68@mcmaster.ca>
Message-ID: <1502112324.14660.21.camel@gmail.com>

On ma, 2017-08-07 at 07:49 -0400, Wayde Nie wrote:
> > > proxy_protocol
> > > on the listen directive I see the correct ip and port picked up
> > > and
> > > logged in the error.log, however, then nginx stops sending the
> > > smtp
> > > greeting... My mail client connects to my loadbalancer, the lb
> > > connects
> > > to nginx:587 sending the PROXY line, nginx parses and logs the
> > > PROXY
> > > fields,??then the client times out waiting for any return traffic
> > > from
> > > nginx... I know it's something I'm doing :-)
> > > 
> > > I'm happy to keep poking away at it, but curious mostly, if you
> > > think
> > > the approach is sound? (ie. use $proxy_protocol_addr, set by
> > > proxy_protocol directive and pass in to auth_http script in auth
> > > url
> > > as
> > > a get param?) and if an initial patch that starts by just setting
> > > $proxy_protocol_* variables would be a useful first step in this
> > > type
> > > of
> > > functionality?
> > > > Just to get the picture right (it looks to me that your
> > > > downstream smtp
> > > > server expects the proxy protocol), what's the exact flow
> > > > you're trying
> > > > to accomplish?
> > > > 
> ?
> ?Hi Kees,
> 
> Thanks for looking!
> 
> In my use case I have an external hardware loadbalancer that is
> receiving end user connections on a VIP, pre-pending proxy_protocol
> header and loadbalancing them to small pool of nginx servers running
> as the mail proxies. Nginx is parsing the proxy_protocol header and
> (I hope) proxying to my upstream smtp server without passing the
> proxy_protocol header, which my upstream smtp server doesn't support
> (as currently implemented).
> 
> Flow like:
> 1) Client makes connection to [external-LB-VIP:587] for email
> submission
> 2) [external-LB-VIP:587] --> injects proxy_protocol header --> load
> balances to set of nginx services via TCP service pool (ie: lb
> straight tcp, no application level inspection by loadbalancer, other
> than prepending proxy_protocol header)
> 3) Nginx parses proxy_protocol header, logs client ip and passes
> client IP into auth_http script, along with username and password for
> authn/authz
> 4) on successful return response from auth_http; nginx proxies mail
> submission to upstream smtp server without proxy_protocol header.
> 
> So, if possible, I'd like nginx to get the client ip passed to it
> from the external hardware load balancer, log it and then use it in
> the auth_http script, but otherwise not pass it on to the upstream
> smtp server...
> 
> Is this doable?
> 
> Thanks,Wayde.??
Yep. Should be.
I just noticed that my mail from Jul 20 is emtpy and should contain the
latest patch. I've just sent the patch.
In your case, the config should probably be something like:
mail {
????server_name mail.example.com;
????auth_http???<auth url>;
????server {
????????listen 587 proxy_protocol;
????????protocol smtp;
????????set_real_ip_from <external lb ip>;
????}
}

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170807/937ee340/attachment-0001.html>

From m at rkellis.com  Mon Aug  7 15:15:33 2017
From: m at rkellis.com (Mark Ellis)
Date: Mon, 07 Aug 2017 16:15:33 +0100
Subject: [PATCH] Allows hyphens in variable names so can access hyphenated
 query vars
In-Reply-To: <20170807132049.GZ93611@mdounin.ru>
References: <08008b0d1f4ffbd017d9.1501673411@ma.h1.thisaintnews.com>
 <20170807132049.GZ93611@mdounin.ru>
Message-ID: <1502118933.3126950.1065708040.28804A22@webmail.messagingengine.com>

Ok. How would you suggest accessing query parameters with hyphens?

e.g. `?foo-bar=123`

how would you get the value of foo-bar?

Thanks

On Mon, 7 Aug 2017, at 02:20 PM, Maxim Dounin wrote:
> Hello!
> 
> On Wed, Aug 02, 2017 at 11:30:11AM +0000, Mark Ellis via nginx-devel
> wrote:
> 
> > # HG changeset patch
> > # User Mark Ellis <m at rkellis.com>
> > # Date 1501670303 -3600
> > #      Wed Aug 02 11:38:23 2017 +0100
> > # Node ID 08008b0d1f4ffbd017d9f50142bdd46d7c4a41d2
> > # Parent  1eb753aa8e5e9ee5059ebb544194c8919b674873
> > Allows hyphens in variable names so can access hyphenated query vars
> > 
> > This patch allows hyphens in variable names so that you can access query string
> > parameters with hyphens
> > 
> > Before if you had a query string param such as `?foo-bar=123` you could not
> > access it with the variable `$foo-bar` as variables could not contain hyphens.
> > 
> > diff -r 1eb753aa8e5e -r 08008b0d1f4f src/http/ngx_http_script.c
> > --- a/src/http/ngx_http_script.c	Tue Aug 01 19:12:10 2017 +0300
> > +++ b/src/http/ngx_http_script.c	Wed Aug 02 11:38:23 2017 +0100
> > @@ -403,6 +403,7 @@
> >                  if ((ch >= 'A' && ch <= 'Z')
> >                      || (ch >= 'a' && ch <= 'z')
> >                      || (ch >= '0' && ch <= '9')
> > +                    || ch == '-'
> >                      || ch == '_')
> >                  {
> >                      continue;
> 
> No, thanks.  Such a change will break a lot of configs, not to 
> mention it is counter-intuitive and contradicts how things usually 
> work in other programming languages.
> 
> -- 
> Maxim Dounin
> http://nginx.org/

From mdounin at mdounin.ru  Mon Aug  7 15:46:55 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 7 Aug 2017 18:46:55 +0300
Subject: [PATCH] Allows hyphens in variable names so can access hyphenated
 query vars
In-Reply-To: <1502118933.3126950.1065708040.28804A22@webmail.messagingengine.com>
References: <08008b0d1f4ffbd017d9.1501673411@ma.h1.thisaintnews.com>
 <20170807132049.GZ93611@mdounin.ru>
 <1502118933.3126950.1065708040.28804A22@webmail.messagingengine.com>
Message-ID: <20170807154655.GE93611@mdounin.ru>

Hello!

On Mon, Aug 07, 2017 at 04:15:33PM +0100, Mark Ellis via nginx-devel wrote:

> Ok. How would you suggest accessing query parameters with hyphens?
> 
> e.g. `?foo-bar=123`
> 
> how would you get the value of foo-bar?

It is more or less trivial to extract anything from $args using a 
regular expression.  And the same applies to things like 
"foo$bar=123", or any other fancy character you want to use in the 
query string.

Alternative solution would be to avoid such naming, and use 
aphanumeric plus "_" as directly accessible via $arg_* variables.

-- 
Maxim Dounin
http://nginx.org/

From mdounin at mdounin.ru  Mon Aug  7 22:04:38 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 8 Aug 2017 01:04:38 +0300
Subject: Supporting the Forwarded header (RFC 7239)
In-Reply-To: <CALHHdhwaEXSftBiCUO06V6e6c0F=Wwpw+6NX5cyQ6U_F+PuKow@mail.gmail.com>
References: <CALHHdhwaEXSftBiCUO06V6e6c0F=Wwpw+6NX5cyQ6U_F+PuKow@mail.gmail.com>
Message-ID: <20170807220438.GH93611@mdounin.ru>

Hello!

On Sun, Aug 06, 2017 at 09:11:03PM +0300, Vasiliy Faronov wrote:

> Hi nginx developers,
> 
> As you know, RFC 7239 defines a Forwarded header to replace the zoo of
> X-Forwarded-* with a single extensible syntax.
> 
> There seems to be growing interest in deploying Forwarded. For
> example, aiohttp, a popular Python library, recently started reading
> Forwarded by default.
> 
> I'd like to see a $proxy_add_forwarded variable in nginx -- similar in
> essence to $proxy_add_x_forwarded_for -- and I'd like to try and
> contribute a patch.
> 
> Can you tell me if this sounds like a good idea to you, and if yes,
> which of the following approaches you prefer? (I'm sorry for the wall
> of text)

We've previously considered adding Forwarded support, though 
postponded it as it seems to be somewhat different from 
X-Forwarded-For / X-Real-IP we do support, and we haven't seen any 
practial implementations.

> 1. $proxy_add_forwarded blindly appends a "for=..." after a comma,
> just like $proxy_add_x_forwarded_for does. (This cannot be emulated in
> config because of ticket #1316; also, nginx's $remote_addr is not
> quite in the format required by RFC 7239.)
> 
> The problem here: suppose an external user sends:
> 
>     Forwarded: for=injected;by="
> 
> If you just append to this, you get:
> 
>     Forwarded: for=injected;by=", for=real
> 
> This puts the burden on the receiving application to parse this
> robustly and recover "for=real". They can't just split on comma
> (because comma can occur inside a valid quoted-string), they need more
> logic. It's doable, they just have to be aware of the problem and
> actively mitigate it.

Certainly blindly adding a value without checking the header 
syntax looks like a bad idea.  It opens an obvious vulnerability 
as RFC-complain parsing by the upstream server will produce an 
incorrect "for=injected" result.

It is also not clear how to deal with such syntactically incorrect 
headers: removing the previous headers will obviously result in an 
information loss, while "fixing" them also looks wrong.

>From this point of view, X-Forwarded-For is much better, as it 
does not require to parse anything.

> 2. Parse any incoming Forwarded headers into an internal
> representation, then re-serialize it with an added element.
> 
> This is obviously more expensive. But, if you think that supporting
> Forwarded is a good idea, then eventually you want to support it in
> ngx_http_realip_module and wherever else nginx looks at X-Forwarded-*.
> Then you need to parse anyway.

This looks like an overkill to me, especially considering that 
Forwarded can have arbitrary extensions.  

> 3. Validate the syntax of any incoming Forwarded headers with a regex.
> If they are valid, append to them. If they are invalid, replace them
> with a single "for=unknown" (explicitly allowed by RFC 7239), and
> append to that.

I don't see how this is explicitly allowed by RFC 7239.
And anyway this is an information loss, see above.

> 4. Do any of the above, but in a third-party module, where one could
> experiment more freely.
> 
> I think requiring a third-party module to support Forwarded will just
> lead to everybody sticking with X-Forwarded-*, or else trying to
> emulate it in config with poor results. It's just not enough of a
> feature on its own.

It should be more or less trivial to implement config-based 
emulation of $proxy_add_forwarded, using map{} and appropriate 
regular expression checks.  The main problem here is ticket #1316, 
yet it probably should be addressed separately.

On the other hand, such approach also faces the problem on 
what to do with syntactically invalid Forwarded headers, and I 
don't think I know a solution.

(Also, the interesting part is probably using Forwarded in the 
realip module, and this is certainly can't be done as a 3rd party 
module without re-implementing the whole module.)

> Then again, maybe RFC 7239 was a bad idea and everybody *should* just
> stick to X-Forwarded-*.

I think _eventually_ there should be some standard on this, and 
RFC 7239 looks like a step in the right direction.  It seems to 
have various problems though, and it may be a good idea to 
postpone the implementation till it is clear how to address these 
problems and/or an updated RFC to address them is available.

-- 
Maxim Dounin
http://nginx.org/

From niew at mcmaster.ca  Tue Aug  8 06:50:29 2017
From: niew at mcmaster.ca (Wayde Nie)
Date: Tue, 8 Aug 2017 02:50:29 -0400
Subject: [PATCH] Add proxy_protocol option to mail listener
In-Reply-To: <1502112324.14660.21.camel@gmail.com>
References: <ca09cf40-7f60-e63a-8957-d14f0bb297d6@mcmaster.ca>
 <1502091868.14660.6.camel@gmail.com>
 <8f8e9d67-191a-92b1-990d-bdde41a5fe68@mcmaster.ca>
 <1502112324.14660.21.camel@gmail.com>
Message-ID: <54c7868f-e0bc-d6ab-3a05-a89238c0ccba@mcmaster.ca>

Thanks Kees and Maxim, for the patch and the discussion. A very helpful
community!

This looks like it's working well for my use case for a tcp465/"ssl on"
mail proxy server block. My earlier issues appear to have (at least
partly) been related to STARTTLS with a tcp587/"starttls only" block
that I started my investigations with...

Once I switched over to the straight SSL block things started working a
lot better. I'll get back to my STARTTLS challenge in a little bit,
unless there's some reason you know of off-hand that it should act
differently?

The only other thing I've noticed is an inconsistency/typo in the
'set_real_ip_from' directive (as in the documentation section of the
patchset). It looks like it is actually parsed as 'set_realip_from' (no
_ in the realip part). I'm not sure which is the correct form ( :-) ),
but the feature works when I use the 'set_realip_from' form , so it does
differ from the docs...

Thanks again Kees! Wayde.



On 08/07/2017 09:25 AM, Kees Bos wrote:
> On ma, 2017-08-07 at 07:49 -0400, Wayde Nie wrote:
>> On 08/07/2017 03:44 AM, Kees Bos wrote:, and when I do set
>>>> proxy_protocol
>>>> on the listen directive I see the correct ip and port picked up and
>>>> logged in the error.log, however, then nginx stops sending the smtp
>>>> greeting... My mail client connects to my loadbalancer, the lb
>>>> connects
>>>> to nginx:587 sending the PROXY line, nginx parses and logs the PROXY
>>>> fields,  then the client times out waiting for any return traffic
>>>> from
>>>> nginx... I know it's something I'm doing :-)
>>>>
>>>> I'm happy to keep poking away at it, but curious mostly, if you think
>>>> the approach is sound? (ie. use $proxy_protocol_addr, set by
>>>> proxy_protocol directive and pass in to auth_http script in auth url
>>>> as
>>>> a get param?) and if an initial patch that starts by just setting
>>>> $proxy_protocol_* variables would be a useful first step in this type
>>>> of
>>>> functionality?
>>>>> Just to get the picture right (it looks to me that your downstream smtp
>>>>> server expects the proxy protocol), what's the exact flow you're trying
>>>>> to accomplish?
>>
>>  Hi Kees,
>>
>> Thanks for looking!
>>
>> In my use case I have an external hardware loadbalancer that is
>> receiving end user connections on a VIP, pre-pending proxy_protocol
>> header and loadbalancing them to small pool of nginx servers running
>> as the mail proxies. Nginx is parsing the proxy_protocol header and
>> (I hope) proxying to my upstream smtp server without passing the
>> proxy_protocol header, which my upstream smtp server doesn't support
>> (as currently implemented).
>>
>> Flow like:
>>
>>     1) Client makes connection to [external-LB-VIP:587] for email
>>     submission
>>     2) [external-LB-VIP:587] --> injects proxy_protocol header -->
>>     load balances to set of nginx services via TCP service pool (ie:
>>     lb straight tcp, no application level inspection by loadbalancer,
>>     other than prepending proxy_protocol header)
>>     3) Nginx parses proxy_protocol header, logs client ip and passes
>>     client IP into auth_http script, along with username and password
>>     for authn/authz
>>     4) on successful return response from auth_http; nginx proxies
>>     mail submission to upstream smtp server without proxy_protocol
>>     header.
>>
>>
>> So, if possible, I'd like nginx to get the client ip passed to it
>> from the external hardware load balancer, log it and then use it in
>> the auth_http script, but otherwise not pass it on to the upstream
>> smtp server...
>>
>> Is this doable?
>>
>> Thanks,Wayde. 
>
> Yep. Should be.
>
> I just noticed that my mail from Jul 20 is emtpy and should contain
> the latest patch. I've just sent the patch.
>
>
> In your case, the config should probably be something like:
>
> mail {
>     server_name mail.example.com;
>     auth_http   <auth url>;
>     server {
>         listen 587 proxy_protocol;
>         protocol smtp;
>         set_real_ip_from <external lb ip>;
>     }
> }
>
>
>
>
>
>
>
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170808/326c2b3a/attachment.html>

From vfaronov at gmail.com  Tue Aug  8 08:52:58 2017
From: vfaronov at gmail.com (Vasiliy Faronov)
Date: Tue, 8 Aug 2017 11:52:58 +0300
Subject: Supporting the Forwarded header (RFC 7239)
In-Reply-To: <20170807220438.GH93611@mdounin.ru>
References: <CALHHdhwaEXSftBiCUO06V6e6c0F=Wwpw+6NX5cyQ6U_F+PuKow@mail.gmail.com>
 <20170807220438.GH93611@mdounin.ru>
Message-ID: <CALHHdhyNjN2PJh6j2JrKzYu7krCmBj7ibZvt3ZcOo4brg+3=ng@mail.gmail.com>

Hi Maxim,

Thank you for your reply.


> I don't see how this is explicitly allowed by RFC 7239.

In Section 4:

   A proxy MAY remove all "Forwarded" header fields from a request.

And in Section 6.2:

   The "unknown" identifier is used when the identity of the preceding
   entity is not known, but the proxy server still wants to signal that
   a forwarding of the request was made.


> And anyway this is an information loss, see above.

Isn't it acceptable to lose information from a badly malformed header?
An upstream probably wouldn't be able to extract that information
anyway, so this should only affect upstream debug/logging.


I understand your general misgivings, though.


> It should be more or less trivial to implement config-based
> emulation of $proxy_add_forwarded, using map{} and appropriate
> regular expression checks.  The main problem here is ticket #1316,
> yet it probably should be addressed separately.

Until such a time as #1316 is dealt with, do you think it would be a
good idea to explicitly set Forwarded to
ngx_http_process_multi_header_lines, like X-Forwarded-For?


> On the other hand, such approach also faces the problem on
> what to do with syntactically invalid Forwarded headers, and I
> don't think I know a solution.

Any reasonable implementation of Forwarded on the upstream must be
prepared to deal with such malformed headers anyway. (For example,
aiohttp is.) So, in cases where the upstream is known to do this, the
config-based approach could be workable.


-- 
Vasiliy

From mdounin at mdounin.ru  Tue Aug  8 10:41:47 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 8 Aug 2017 13:41:47 +0300
Subject: Supporting the Forwarded header (RFC 7239)
In-Reply-To: <CALHHdhyNjN2PJh6j2JrKzYu7krCmBj7ibZvt3ZcOo4brg+3=ng@mail.gmail.com>
References: <CALHHdhwaEXSftBiCUO06V6e6c0F=Wwpw+6NX5cyQ6U_F+PuKow@mail.gmail.com>
 <20170807220438.GH93611@mdounin.ru>
 <CALHHdhyNjN2PJh6j2JrKzYu7krCmBj7ibZvt3ZcOo4brg+3=ng@mail.gmail.com>
Message-ID: <20170808104147.GJ93611@mdounin.ru>

Hello!

On Tue, Aug 08, 2017 at 11:52:58AM +0300, Vasiliy Faronov wrote:

> Hi Maxim,
> 
> Thank you for your reply.
> 
> 
> > I don't see how this is explicitly allowed by RFC 7239.
> 
> In Section 4:
> 
>    A proxy MAY remove all "Forwarded" header fields from a request.

Obviously enough, there are a lot of cases when proxy may want to 
remove Forwarded headers, including ones explicitly specified in 
the Section 7.2 (Header Field Preservation).

> And in Section 6.2:
> 
>    The "unknown" identifier is used when the identity of the preceding
>    entity is not known, but the proxy server still wants to signal that
>    a forwarding of the request was made.

As long as Forwarded header is not trusted, we never know if a 
forwarding of the request was mad or not.  Section 6.2 clearly 
talks about the case when we know that forwarding was made, but 
don't know the address.

> > And anyway this is an information loss, see above.
> 
> Isn't it acceptable to lose information from a badly malformed header?
> An upstream probably wouldn't be able to extract that information
> anyway, so this should only affect upstream debug/logging.

It may or may not be acceptable.  For example, this information 
can be further used to detect and/or analyze attacks, and/or 
identify requests during abuse investigation.

[...]

> > It should be more or less trivial to implement config-based
> > emulation of $proxy_add_forwarded, using map{} and appropriate
> > regular expression checks.  The main problem here is ticket #1316,
> > yet it probably should be addressed separately.
> 
> Until such a time as #1316 is dealt with, do you think it would be a
> good idea to explicitly set Forwarded to
> ngx_http_process_multi_header_lines, like X-Forwarded-For?

I don't really think that using explicit 
ngx_http_process_multi_header_lines() is a good solution, as there 
can be additional headers we don't handle explicitly.  Rather, I 
would like to try to address this in general for all unknown 
headers.

> > On the other hand, such approach also faces the problem on
> > what to do with syntactically invalid Forwarded headers, and I
> > don't think I know a solution.
> 
> Any reasonable implementation of Forwarded on the upstream must be
> prepared to deal with such malformed headers anyway. (For example,
> aiohttp is.) So, in cases where the upstream is known to do this, the
> config-based approach could be workable.

Well, config-based approach could be easily used to strip 
malformed headers.  The problem here is how to avoid information loss 
as discussed above.

-- 
Maxim Dounin
http://nginx.org/

From ru at nginx.com  Tue Aug  8 12:16:23 2017
From: ru at nginx.com (Ruslan Ermilov)
Date: Tue, 08 Aug 2017 12:16:23 +0000
Subject: [nginx] Slab: fixed initialization on win32.
Message-ID: <hg.69f9ee0342db.1502194583.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/69f9ee0342db
branches:  
changeset: 7081:69f9ee0342db
user:      Ruslan Ermilov <ru at nginx.com>
date:      Tue Aug 08 15:16:01 2017 +0300
description:
Slab: fixed initialization on win32.

On Windows, a worker process does not call ngx_slab_init() from
ngx_init_zone_pool(), so ngx_slab_max_size, ngx_slab_exact_size,
and ngx_slab_exact_shift were left uninitialized.

diffstat:

 src/core/nginx.c    |   6 ++++++
 src/core/ngx_slab.c |  23 +++++++++++++----------
 src/core/ngx_slab.h |   1 +
 3 files changed, 20 insertions(+), 10 deletions(-)

diffs (67 lines):

diff -r ac120e797d28 -r 69f9ee0342db src/core/nginx.c
--- a/src/core/nginx.c	Fri Aug 04 17:03:10 2017 +0300
+++ b/src/core/nginx.c	Tue Aug 08 15:16:01 2017 +0300
@@ -273,6 +273,12 @@ main(int argc, char *const *argv)
         return 1;
     }
 
+    /*
+     * ngx_slab_sizes_init() requires ngx_pagesize set in ngx_os_init()
+     */
+
+    ngx_slab_sizes_init();
+
     if (ngx_add_inherited_sockets(&init_cycle) != NGX_OK) {
         return 1;
     }
diff -r ac120e797d28 -r 69f9ee0342db src/core/ngx_slab.c
--- a/src/core/ngx_slab.c	Fri Aug 04 17:03:10 2017 +0300
+++ b/src/core/ngx_slab.c	Tue Aug 08 15:16:01 2017 +0300
@@ -83,6 +83,19 @@ static ngx_uint_t  ngx_slab_exact_shift;
 
 
 void
+ngx_slab_sizes_init(void)
+{
+    ngx_uint_t  n;
+
+    ngx_slab_max_size = ngx_pagesize / 2;
+    ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
+    for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
+        /* void */
+    }
+}
+
+
+void
 ngx_slab_init(ngx_slab_pool_t *pool)
 {
     u_char           *p;
@@ -91,16 +104,6 @@ ngx_slab_init(ngx_slab_pool_t *pool)
     ngx_uint_t        i, n, pages;
     ngx_slab_page_t  *slots, *page;
 
-    /* STUB */
-    if (ngx_slab_max_size == 0) {
-        ngx_slab_max_size = ngx_pagesize / 2;
-        ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
-        for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
-            /* void */
-        }
-    }
-    /**/
-
     pool->min_size = (size_t) 1 << pool->min_shift;
 
     slots = ngx_slab_slots(pool);
diff -r ac120e797d28 -r 69f9ee0342db src/core/ngx_slab.h
--- a/src/core/ngx_slab.h	Fri Aug 04 17:03:10 2017 +0300
+++ b/src/core/ngx_slab.h	Tue Aug 08 15:16:01 2017 +0300
@@ -59,6 +59,7 @@ typedef struct {
 } ngx_slab_pool_t;
 
 
+void ngx_slab_sizes_init(void);
 void ngx_slab_init(ngx_slab_pool_t *pool);
 void *ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size);
 void *ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size);

From mdounin at mdounin.ru  Tue Aug  8 14:46:30 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 08 Aug 2017 14:46:30 +0000
Subject: [nginx] Updated PCRE used for win32 builds.
Message-ID: <hg.74431027af26.1502203590.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/74431027af26
branches:  
changeset: 7082:74431027af26
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Tue Aug 08 15:21:10 2017 +0300
description:
Updated PCRE used for win32 builds.

diffstat:

 misc/GNUmakefile |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (12 lines):

diff --git a/misc/GNUmakefile b/misc/GNUmakefile
--- a/misc/GNUmakefile
+++ b/misc/GNUmakefile
@@ -8,7 +8,7 @@ CC =		cl
 OBJS =		objs.msvc8
 OPENSSL =	openssl-1.0.2l
 ZLIB =		zlib-1.2.11
-PCRE =		pcre-8.40
+PCRE =		pcre-8.41
 
 
 release: export

From mdounin at mdounin.ru  Tue Aug  8 15:06:43 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 08 Aug 2017 15:06:43 +0000
Subject: [nginx] nginx-1.13.4-RELEASE
Message-ID: <hg.bbc642c813c8.1502204803.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/bbc642c813c8
branches:  
changeset: 7083:bbc642c813c8
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Tue Aug 08 18:00:11 2017 +0300
description:
nginx-1.13.4-RELEASE

diffstat:

 docs/xml/nginx/changes.xml |  65 ++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 65 insertions(+), 0 deletions(-)

diffs (75 lines):

diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml
--- a/docs/xml/nginx/changes.xml
+++ b/docs/xml/nginx/changes.xml
@@ -5,6 +5,71 @@
 <change_log title="nginx">
 
 
+<changes ver="1.13.4" date="2017-08-08">
+
+<change type="feature">
+<para lang="ru">
+?????? ngx_http_mirror_module.
+</para>
+<para lang="en">
+the ngx_http_mirror_module.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+?????????? ?????????? ????? ???????????? ??? ???????????? ????????????,
+???? ????????????? ???????? reuseport ????????? listen ?? Linux.
+</para>
+<para lang="en">
+client connections might be dropped during configuration testing
+when using the "reuseport" parameter of the "listen" directive on Linux.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+???? ??????? ????? ???? ?????????? ? ???????????,
+???? ??? ???? ????????? ? ???? ? ?????????????? ?????????????.
+</para>
+<para lang="en">
+request body might not be available in subrequests
+if it was saved to a file and proxying was used.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+??????? ???? ?? max_size ?? ???????? ?? Windows.
+</para>
+<para lang="en">
+cleaning cache based on the "max_size" parameter did not work on Windows.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+????? ????????? ??????????? ?????? ?? Windows ????????? 4096 ???? ??????.
+</para>
+<para lang="en">
+any shared memory allocation required 4096 bytes on Windows.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+??? ????????????? ????????? zone ? ????? upstream ?? Windows
+??????? ??????? ??? ??????????? ????????.
+</para>
+<para lang="en">
+nginx worker might be terminated abnormally
+when using the "zone" directive inside the "upstream" block on Windows.
+</para>
+</change>
+
+</changes>
+
+
 <changes ver="1.13.3" date="2017-07-11">
 
 <change type="security">

From mdounin at mdounin.ru  Tue Aug  8 15:06:46 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 08 Aug 2017 15:06:46 +0000
Subject: [nginx] release-1.13.4 tag
Message-ID: <hg.2f48ab272052.1502204806.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/2f48ab272052
branches:  
changeset: 7084:2f48ab272052
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Tue Aug 08 18:00:11 2017 +0300
description:
release-1.13.4 tag

diffstat:

 .hgtags |  1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diffs (8 lines):

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -416,3 +416,4 @@ 3671096a45bce570a2afa20b9faf42c7fb0f7e66
 539f7893ecb96bee60965528c8958d7eb2f1ce6b release-1.13.1
 5be2b25bdc65775a85f18f68a4be4f58c7384415 release-1.13.2
 8457ce87640f9bfe6221c4ac4466ced20e03bebe release-1.13.3
+bbc642c813c829963ce8197c0ca237ab7601f3d4 release-1.13.4

From igor at sysoev.ru  Tue Aug  8 15:07:25 2017
From: igor at sysoev.ru (Igor Sysoev)
Date: Tue, 08 Aug 2017 15:07:25 +0000
Subject: [njs] Style fixes.
Message-ID: <hg.e2baf506be4c.1502204845.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/e2baf506be4c
branches:  
changeset: 391:e2baf506be4c
user:      Igor Sysoev <igor at sysoev.ru>
date:      Tue Aug 08 17:47:53 2017 +0300
description:
Style fixes.

diffstat:

 njs/njs_generator.c |  5 +----
 njs/njs_variable.c  |  3 ---
 2 files changed, 1 insertions(+), 7 deletions(-)

diffs (59 lines):

diff -r c347ea0bbdd6 -r e2baf506be4c njs/njs_generator.c
--- a/njs/njs_generator.c	Tue Jul 18 19:25:50 2017 +0300
+++ b/njs/njs_generator.c	Tue Aug 08 17:47:53 2017 +0300
@@ -380,7 +380,7 @@ njs_generate_builtin_object(njs_vm_t *vm
 
     node->index = njs_generator_dest_index(vm, parser, node);
     if (nxt_slow_path(node->index == NJS_INDEX_ERROR)) {
-        return node->index;
+        return NXT_ERROR;
     }
 
     njs_generate_code(parser, njs_vmcode_object_copy_t, copy);
@@ -2541,13 +2541,11 @@ njs_generator_temp_index_get(njs_vm_t *v
     }
 
     if (vm->accumulative && scope->type == NJS_SCOPE_GLOBAL) {
-
         /*
          * When non-clonable VM runs in accumulative mode
          * all global variables are allocated in absolute scope
          * to simplify global scope handling.
          */
-
         value = nxt_mem_cache_align(vm->mem_cache_pool, sizeof(njs_value_t),
                                     sizeof(njs_value_t));
         if (nxt_slow_path(value == NULL)) {
@@ -2557,7 +2555,6 @@ njs_generator_temp_index_get(njs_vm_t *v
         index = (njs_index_t) value;
 
     } else {
-
         value = nxt_array_add(scope->values[0], &njs_array_mem_proto,
                               vm->mem_cache_pool);
         if (nxt_slow_path(value == NULL)) {
diff -r c347ea0bbdd6 -r e2baf506be4c njs/njs_variable.c
--- a/njs/njs_variable.c	Tue Jul 18 19:25:50 2017 +0300
+++ b/njs/njs_variable.c	Tue Aug 08 17:47:53 2017 +0300
@@ -356,13 +356,11 @@ njs_variable_get(njs_vm_t *vm, njs_parse
     }
 
     if (vm->accumulative && vs.scope->type == NJS_SCOPE_GLOBAL) {
-
         /*
          * When non-clonable VM runs in accumulative mode all
          * global variables should be allocated in absolute scope
          * to share them among consecutive VM invocations.
          */
-
         value = nxt_mem_cache_align(vm->mem_cache_pool, sizeof(njs_value_t),
                                     sizeof(njs_value_t));
         if (nxt_slow_path(value == NULL)) {
@@ -391,7 +389,6 @@ njs_variable_get(njs_vm_t *vm, njs_parse
 
         index = vs.scope->next_index[n];
         vs.scope->next_index[n] += sizeof(njs_value_t);
-
     }
 
     if (njs_is_object(&var->value)) {

From igor at sysoev.ru  Tue Aug  8 15:07:26 2017
From: igor at sysoev.ru (Igor Sysoev)
Date: Tue, 08 Aug 2017 15:07:26 +0000
Subject: [njs] Removed unused field.
Message-ID: <hg.bc306c37d037.1502204846.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/bc306c37d037
branches:  
changeset: 392:bc306c37d037
user:      Igor Sysoev <igor at sysoev.ru>
date:      Tue Aug 08 17:47:55 2017 +0300
description:
Removed unused field.

diffstat:

 njs/njs_parser.c            |  12 ------------
 njs/njs_parser.h            |   2 --
 njs/njs_parser_expression.c |   2 --
 3 files changed, 0 insertions(+), 16 deletions(-)

diffs (81 lines):

diff -r e2baf506be4c -r bc306c37d037 njs/njs_parser.c
--- a/njs/njs_parser.c	Tue Aug 08 17:47:53 2017 +0300
+++ b/njs/njs_parser.c	Tue Aug 08 17:47:55 2017 +0300
@@ -934,8 +934,6 @@ njs_parser_if_statement(njs_vm_t *vm, nj
     njs_token_t        token;
     njs_parser_node_t  *node, *cond, *stmt;
 
-    parser->branch = 1;
-
     token = njs_parser_grouping_expression(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
@@ -995,8 +993,6 @@ njs_parser_switch_statement(njs_vm_t *vm
     njs_token_t        token;
     njs_parser_node_t  *node, *swtch, *branch, *dflt, **last;
 
-    parser->branch = 1;
-
     node = NULL;
     branch = NULL;
     dflt = NULL;
@@ -1114,8 +1110,6 @@ njs_parser_while_statement(njs_vm_t *vm,
     njs_token_t        token;
     njs_parser_node_t  *node, *cond;
 
-    parser->branch = 1;
-
     token = njs_parser_grouping_expression(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
@@ -1150,8 +1144,6 @@ njs_parser_do_while_statement(njs_vm_t *
     njs_token_t        token;
     njs_parser_node_t  *node, *stmt;
 
-    parser->branch = 1;
-
     token = njs_parser_token(parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
@@ -1195,8 +1187,6 @@ njs_parser_for_statement(njs_vm_t *vm, n
     njs_token_t        token;
     njs_parser_node_t  *node, *init, *condition, *update, *cond, *body;
 
-    parser->branch = 1;
-
     init = NULL;
     condition = NULL;
     update = NULL;
@@ -1580,8 +1570,6 @@ njs_parser_try_statement(njs_vm_t *vm, n
     njs_variable_t     *var;
     njs_parser_node_t  *node, *try, *catch;
 
-    parser->branch = 1;
-
     token = njs_parser_try_block(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
diff -r e2baf506be4c -r bc306c37d037 njs/njs_parser.h
--- a/njs/njs_parser.h	Tue Aug 08 17:47:53 2017 +0300
+++ b/njs/njs_parser.h	Tue Aug 08 17:47:55 2017 +0300
@@ -314,8 +314,6 @@ struct njs_parser_s {
 
     nxt_array_t                     *index_cache;
 
-    uint8_t                         branch;       /* 1 bit */
-
     /* Parsing Function() or eval(). */
     uint8_t                         runtime;      /* 1 bit */
 
diff -r e2baf506be4c -r bc306c37d037 njs/njs_parser_expression.c
--- a/njs/njs_parser_expression.c	Tue Aug 08 17:47:53 2017 +0300
+++ b/njs/njs_parser_expression.c	Tue Aug 08 17:47:55 2017 +0300
@@ -518,8 +518,6 @@ njs_parser_conditional_expression(njs_vm
             return token;
         }
 
-        parser->branch = 1;
-
         token = njs_parser_token(parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;

From igor at sysoev.ru  Tue Aug  8 15:07:28 2017
From: igor at sysoev.ru (Igor Sysoev)
Date: Tue, 08 Aug 2017 15:07:28 +0000
Subject: [njs] Style fix in Unicode header files and Perl scripts which
Message-ID: <hg.49a8c7cfb121.1502204848.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/49a8c7cfb121
branches:  
changeset: 393:49a8c7cfb121
user:      Igor Sysoev <igor at sysoev.ru>
date:      Tue Aug 08 17:47:56 2017 +0300
description:
Style fix in Unicode header files and Perl scripts which
generate the files.

diffstat:

 nxt/nxt_unicode_lower_case.h  |  108 ++++++++++++++++++++++++++++---------
 nxt/nxt_unicode_lower_case.pl |    9 ++-
 nxt/nxt_unicode_upper_case.h  |  120 +++++++++++++++++++++++++++++++----------
 nxt/nxt_unicode_upper_case.pl |    8 ++-
 4 files changed, 183 insertions(+), 62 deletions(-)

diffs (684 lines):

diff -r bc306c37d037 -r 49a8c7cfb121 nxt/nxt_unicode_lower_case.h
--- a/nxt/nxt_unicode_lower_case.h	Tue Aug 08 17:47:55 2017 +0300
+++ b/nxt/nxt_unicode_lower_case.h	Tue Aug 08 17:47:56 2017 +0300
@@ -9,7 +9,9 @@
 #define NXT_UNICODE_BLOCK_SIZE      128
 
 
-static const uint32_t  nxt_unicode_lower_case_block_000[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_000[128]
+    nxt_aligned(64) =
+{
     0x00000, 0x00001, 0x00002, 0x00003, 0x00004, 0x00005, 0x00006, 0x00007,
     0x00008, 0x00009, 0x0000a, 0x0000b, 0x0000c, 0x0000d, 0x0000e, 0x0000f,
     0x00010, 0x00011, 0x00012, 0x00013, 0x00014, 0x00015, 0x00016, 0x00017,
@@ -29,7 +31,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_001[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_001[128]
+    nxt_aligned(64) =
+{
     0x00080, 0x00081, 0x00082, 0x00083, 0x00084, 0x00085, 0x00086, 0x00087,
     0x00088, 0x00089, 0x0008a, 0x0008b, 0x0008c, 0x0008d, 0x0008e, 0x0008f,
     0x00090, 0x00091, 0x00092, 0x00093, 0x00094, 0x00095, 0x00096, 0x00097,
@@ -49,7 +53,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_002[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_002[128]
+    nxt_aligned(64) =
+{
     0x00101, 0x00101, 0x00103, 0x00103, 0x00105, 0x00105, 0x00107, 0x00107,
     0x00109, 0x00109, 0x0010b, 0x0010b, 0x0010d, 0x0010d, 0x0010f, 0x0010f,
     0x00111, 0x00111, 0x00113, 0x00113, 0x00115, 0x00115, 0x00117, 0x00117,
@@ -69,7 +75,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_003[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_003[128]
+    nxt_aligned(64) =
+{
     0x00180, 0x00253, 0x00183, 0x00183, 0x00185, 0x00185, 0x00254, 0x00188,
     0x00188, 0x00256, 0x00257, 0x0018c, 0x0018c, 0x0018d, 0x001dd, 0x00259,
     0x0025b, 0x00192, 0x00192, 0x00260, 0x00263, 0x00195, 0x00269, 0x00268,
@@ -89,7 +97,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_004[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_004[128]
+    nxt_aligned(64) =
+{
     0x00201, 0x00201, 0x00203, 0x00203, 0x00205, 0x00205, 0x00207, 0x00207,
     0x00209, 0x00209, 0x0020b, 0x0020b, 0x0020d, 0x0020d, 0x0020f, 0x0020f,
     0x00211, 0x00211, 0x00213, 0x00213, 0x00215, 0x00215, 0x00217, 0x00217,
@@ -109,7 +119,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_006[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_006[128]
+    nxt_aligned(64) =
+{
     0x00300, 0x00301, 0x00302, 0x00303, 0x00304, 0x00305, 0x00306, 0x00307,
     0x00308, 0x00309, 0x0030a, 0x0030b, 0x0030c, 0x0030d, 0x0030e, 0x0030f,
     0x00310, 0x00311, 0x00312, 0x00313, 0x00314, 0x00315, 0x00316, 0x00317,
@@ -129,7 +141,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_007[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_007[128]
+    nxt_aligned(64) =
+{
     0x00380, 0x00381, 0x00382, 0x00383, 0x00384, 0x00385, 0x003ac, 0x00387,
     0x003ad, 0x003ae, 0x003af, 0x0038b, 0x003cc, 0x0038d, 0x003cd, 0x003ce,
     0x00390, 0x003b1, 0x003b2, 0x003b3, 0x003b4, 0x003b5, 0x003b6, 0x003b7,
@@ -149,7 +163,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_008[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_008[128]
+    nxt_aligned(64) =
+{
     0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457,
     0x00458, 0x00459, 0x0045a, 0x0045b, 0x0045c, 0x0045d, 0x0045e, 0x0045f,
     0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437,
@@ -169,7 +185,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_009[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_009[128]
+    nxt_aligned(64) =
+{
     0x00481, 0x00481, 0x00482, 0x00483, 0x00484, 0x00485, 0x00486, 0x00487,
     0x00488, 0x00489, 0x0048b, 0x0048b, 0x0048d, 0x0048d, 0x0048f, 0x0048f,
     0x00491, 0x00491, 0x00493, 0x00493, 0x00495, 0x00495, 0x00497, 0x00497,
@@ -189,7 +207,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_00a[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_00a[128]
+    nxt_aligned(64) =
+{
     0x00501, 0x00501, 0x00503, 0x00503, 0x00505, 0x00505, 0x00507, 0x00507,
     0x00509, 0x00509, 0x0050b, 0x0050b, 0x0050d, 0x0050d, 0x0050f, 0x0050f,
     0x00511, 0x00511, 0x00513, 0x00513, 0x00515, 0x00515, 0x00517, 0x00517,
@@ -209,7 +229,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_021[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_021[128]
+    nxt_aligned(64) =
+{
     0x01080, 0x01081, 0x01082, 0x01083, 0x01084, 0x01085, 0x01086, 0x01087,
     0x01088, 0x01089, 0x0108a, 0x0108b, 0x0108c, 0x0108d, 0x0108e, 0x0108f,
     0x01090, 0x01091, 0x01092, 0x01093, 0x01094, 0x01095, 0x01096, 0x01097,
@@ -229,7 +251,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_03c[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_03c[128]
+    nxt_aligned(64) =
+{
     0x01e01, 0x01e01, 0x01e03, 0x01e03, 0x01e05, 0x01e05, 0x01e07, 0x01e07,
     0x01e09, 0x01e09, 0x01e0b, 0x01e0b, 0x01e0d, 0x01e0d, 0x01e0f, 0x01e0f,
     0x01e11, 0x01e11, 0x01e13, 0x01e13, 0x01e15, 0x01e15, 0x01e17, 0x01e17,
@@ -249,7 +273,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_03d[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_03d[128]
+    nxt_aligned(64) =
+{
     0x01e81, 0x01e81, 0x01e83, 0x01e83, 0x01e85, 0x01e85, 0x01e87, 0x01e87,
     0x01e89, 0x01e89, 0x01e8b, 0x01e8b, 0x01e8d, 0x01e8d, 0x01e8f, 0x01e8f,
     0x01e91, 0x01e91, 0x01e93, 0x01e93, 0x01e95, 0x01e95, 0x01e96, 0x01e97,
@@ -269,7 +295,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_03e[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_03e[128]
+    nxt_aligned(64) =
+{
     0x01f00, 0x01f01, 0x01f02, 0x01f03, 0x01f04, 0x01f05, 0x01f06, 0x01f07,
     0x01f00, 0x01f01, 0x01f02, 0x01f03, 0x01f04, 0x01f05, 0x01f06, 0x01f07,
     0x01f10, 0x01f11, 0x01f12, 0x01f13, 0x01f14, 0x01f15, 0x01f16, 0x01f17,
@@ -289,7 +317,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_03f[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_03f[128]
+    nxt_aligned(64) =
+{
     0x01f80, 0x01f81, 0x01f82, 0x01f83, 0x01f84, 0x01f85, 0x01f86, 0x01f87,
     0x01f80, 0x01f81, 0x01f82, 0x01f83, 0x01f84, 0x01f85, 0x01f86, 0x01f87,
     0x01f90, 0x01f91, 0x01f92, 0x01f93, 0x01f94, 0x01f95, 0x01f96, 0x01f97,
@@ -309,7 +339,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_042[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_042[128]
+    nxt_aligned(64) =
+{
     0x02100, 0x02101, 0x02102, 0x02103, 0x02104, 0x02105, 0x02106, 0x02107,
     0x02108, 0x02109, 0x0210a, 0x0210b, 0x0210c, 0x0210d, 0x0210e, 0x0210f,
     0x02110, 0x02111, 0x02112, 0x02113, 0x02114, 0x02115, 0x02116, 0x02117,
@@ -329,7 +361,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_043[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_043[128]
+    nxt_aligned(64) =
+{
     0x02180, 0x02181, 0x02182, 0x02184, 0x02184, 0x02185, 0x02186, 0x02187,
     0x02188, 0x02189, 0x0218a, 0x0218b, 0x0218c, 0x0218d, 0x0218e, 0x0218f,
     0x02190, 0x02191, 0x02192, 0x02193, 0x02194, 0x02195, 0x02196, 0x02197,
@@ -349,7 +383,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_049[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_049[128]
+    nxt_aligned(64) =
+{
     0x02480, 0x02481, 0x02482, 0x02483, 0x02484, 0x02485, 0x02486, 0x02487,
     0x02488, 0x02489, 0x0248a, 0x0248b, 0x0248c, 0x0248d, 0x0248e, 0x0248f,
     0x02490, 0x02491, 0x02492, 0x02493, 0x02494, 0x02495, 0x02496, 0x02497,
@@ -369,7 +405,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_058[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_058[128]
+    nxt_aligned(64) =
+{
     0x02c30, 0x02c31, 0x02c32, 0x02c33, 0x02c34, 0x02c35, 0x02c36, 0x02c37,
     0x02c38, 0x02c39, 0x02c3a, 0x02c3b, 0x02c3c, 0x02c3d, 0x02c3e, 0x02c3f,
     0x02c40, 0x02c41, 0x02c42, 0x02c43, 0x02c44, 0x02c45, 0x02c46, 0x02c47,
@@ -389,7 +427,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_059[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_059[128]
+    nxt_aligned(64) =
+{
     0x02c81, 0x02c81, 0x02c83, 0x02c83, 0x02c85, 0x02c85, 0x02c87, 0x02c87,
     0x02c89, 0x02c89, 0x02c8b, 0x02c8b, 0x02c8d, 0x02c8d, 0x02c8f, 0x02c8f,
     0x02c91, 0x02c91, 0x02c93, 0x02c93, 0x02c95, 0x02c95, 0x02c97, 0x02c97,
@@ -409,7 +449,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_14c[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_14c[128]
+    nxt_aligned(64) =
+{
     0x0a600, 0x0a601, 0x0a602, 0x0a603, 0x0a604, 0x0a605, 0x0a606, 0x0a607,
     0x0a608, 0x0a609, 0x0a60a, 0x0a60b, 0x0a60c, 0x0a60d, 0x0a60e, 0x0a60f,
     0x0a610, 0x0a611, 0x0a612, 0x0a613, 0x0a614, 0x0a615, 0x0a616, 0x0a617,
@@ -429,7 +471,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_14d[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_14d[128]
+    nxt_aligned(64) =
+{
     0x0a681, 0x0a681, 0x0a683, 0x0a683, 0x0a685, 0x0a685, 0x0a687, 0x0a687,
     0x0a689, 0x0a689, 0x0a68b, 0x0a68b, 0x0a68d, 0x0a68d, 0x0a68f, 0x0a68f,
     0x0a691, 0x0a691, 0x0a693, 0x0a693, 0x0a695, 0x0a695, 0x0a697, 0x0a697,
@@ -449,7 +493,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_14e[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_14e[128]
+    nxt_aligned(64) =
+{
     0x0a700, 0x0a701, 0x0a702, 0x0a703, 0x0a704, 0x0a705, 0x0a706, 0x0a707,
     0x0a708, 0x0a709, 0x0a70a, 0x0a70b, 0x0a70c, 0x0a70d, 0x0a70e, 0x0a70f,
     0x0a710, 0x0a711, 0x0a712, 0x0a713, 0x0a714, 0x0a715, 0x0a716, 0x0a717,
@@ -469,7 +515,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_14f[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_14f[128]
+    nxt_aligned(64) =
+{
     0x0a781, 0x0a781, 0x0a783, 0x0a783, 0x0a785, 0x0a785, 0x0a787, 0x0a787,
     0x0a788, 0x0a789, 0x0a78a, 0x0a78c, 0x0a78c, 0x00265, 0x0a78e, 0x0a78f,
     0x0a791, 0x0a791, 0x0a793, 0x0a793, 0x0a794, 0x0a795, 0x0a796, 0x0a797,
@@ -489,7 +537,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_1fe[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_1fe[128]
+    nxt_aligned(64) =
+{
     0x0ff00, 0x0ff01, 0x0ff02, 0x0ff03, 0x0ff04, 0x0ff05, 0x0ff06, 0x0ff07,
     0x0ff08, 0x0ff09, 0x0ff0a, 0x0ff0b, 0x0ff0c, 0x0ff0d, 0x0ff0e, 0x0ff0f,
     0x0ff10, 0x0ff11, 0x0ff12, 0x0ff13, 0x0ff14, 0x0ff15, 0x0ff16, 0x0ff17,
@@ -509,7 +559,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  nxt_unicode_lower_case_block_208[40]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_lower_case_block_208[40]
+    nxt_aligned(64) =
+{
     0x10428, 0x10429, 0x1042a, 0x1042b, 0x1042c, 0x1042d, 0x1042e, 0x1042f,
     0x10430, 0x10431, 0x10432, 0x10433, 0x10434, 0x10435, 0x10436, 0x10437,
     0x10438, 0x10439, 0x1043a, 0x1043b, 0x1043c, 0x1043d, 0x1043e, 0x1043f,
@@ -518,7 +570,9 @@ static const uint32_t  nxt_unicode_lower
 };
 
 
-static const uint32_t  *nxt_unicode_lower_case_blocks[]  nxt_aligned(64) = {
+static const uint32_t  *nxt_unicode_lower_case_blocks[]
+    nxt_aligned(64) =
+{
     nxt_unicode_lower_case_block_000,
     nxt_unicode_lower_case_block_001,
     nxt_unicode_lower_case_block_002,
diff -r bc306c37d037 -r 49a8c7cfb121 nxt/nxt_unicode_lower_case.pl
--- a/nxt/nxt_unicode_lower_case.pl	Tue Aug 08 17:47:55 2017 +0300
+++ b/nxt/nxt_unicode_lower_case.pl	Tue Aug 08 17:47:56 2017 +0300
@@ -56,7 +56,9 @@ for my $block (sort { $a <=> $b } keys %
     my $block_size = ($block != $max_block) ? BLOCK_SIZE : $last_block_size;
 
     print "static const uint32_t  ";
-    printf("nxt_unicode_lower_case_block_%03x[%d]  nxt_aligned(64) = {",
+    printf("nxt_unicode_lower_case_block_%03x[%d]\n" .
+           "    nxt_aligned(64) =\n" .
+           "{",
            $block, $block_size);
 
     for my $c (0 .. $block_size - 1) {
@@ -77,8 +79,9 @@ for my $block (sort { $a <=> $b } keys %
 }
 
 
-print "static const uint32_t  *nxt_unicode_lower_case_blocks[]" .
-      "  nxt_aligned(64) = {\n";
+print "static const uint32_t  *nxt_unicode_lower_case_blocks[]\n" .
+      "    nxt_aligned(64) =\n" .
+      "{\n";
 
 for my $block (0 .. $max_block) {
     if (exists($blocks{$block})) {
diff -r bc306c37d037 -r 49a8c7cfb121 nxt/nxt_unicode_upper_case.h
--- a/nxt/nxt_unicode_upper_case.h	Tue Aug 08 17:47:55 2017 +0300
+++ b/nxt/nxt_unicode_upper_case.h	Tue Aug 08 17:47:56 2017 +0300
@@ -9,7 +9,9 @@
 #define NXT_UNICODE_BLOCK_SIZE       128
 
 
-static const uint32_t  nxt_unicode_upper_case_block_000[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_000[128]
+    nxt_aligned(64) =
+{
     0x00000, 0x00001, 0x00002, 0x00003, 0x00004, 0x00005, 0x00006, 0x00007,
     0x00008, 0x00009, 0x0000a, 0x0000b, 0x0000c, 0x0000d, 0x0000e, 0x0000f,
     0x00010, 0x00011, 0x00012, 0x00013, 0x00014, 0x00015, 0x00016, 0x00017,
@@ -29,7 +31,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_001[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_001[128]
+    nxt_aligned(64) =
+{
     0x00080, 0x00081, 0x00082, 0x00083, 0x00084, 0x00085, 0x00086, 0x00087,
     0x00088, 0x00089, 0x0008a, 0x0008b, 0x0008c, 0x0008d, 0x0008e, 0x0008f,
     0x00090, 0x00091, 0x00092, 0x00093, 0x00094, 0x00095, 0x00096, 0x00097,
@@ -49,7 +53,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_002[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_002[128]
+    nxt_aligned(64) =
+{
     0x00100, 0x00100, 0x00102, 0x00102, 0x00104, 0x00104, 0x00106, 0x00106,
     0x00108, 0x00108, 0x0010a, 0x0010a, 0x0010c, 0x0010c, 0x0010e, 0x0010e,
     0x00110, 0x00110, 0x00112, 0x00112, 0x00114, 0x00114, 0x00116, 0x00116,
@@ -69,7 +75,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_003[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_003[128]
+    nxt_aligned(64) =
+{
     0x00243, 0x00181, 0x00182, 0x00182, 0x00184, 0x00184, 0x00186, 0x00187,
     0x00187, 0x00189, 0x0018a, 0x0018b, 0x0018b, 0x0018d, 0x0018e, 0x0018f,
     0x00190, 0x00191, 0x00191, 0x00193, 0x00194, 0x001f6, 0x00196, 0x00197,
@@ -89,7 +97,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_004[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_004[128]
+    nxt_aligned(64) =
+{
     0x00200, 0x00200, 0x00202, 0x00202, 0x00204, 0x00204, 0x00206, 0x00206,
     0x00208, 0x00208, 0x0020a, 0x0020a, 0x0020c, 0x0020c, 0x0020e, 0x0020e,
     0x00210, 0x00210, 0x00212, 0x00212, 0x00214, 0x00214, 0x00216, 0x00216,
@@ -109,7 +119,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_005[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_005[128]
+    nxt_aligned(64) =
+{
     0x001a6, 0x00281, 0x00282, 0x001a9, 0x00284, 0x00285, 0x00286, 0x00287,
     0x001ae, 0x00244, 0x001b1, 0x001b2, 0x00245, 0x0028d, 0x0028e, 0x0028f,
     0x00290, 0x00291, 0x001b7, 0x00293, 0x00294, 0x00295, 0x00296, 0x00297,
@@ -129,7 +141,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_006[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_006[128]
+    nxt_aligned(64) =
+{
     0x00300, 0x00301, 0x00302, 0x00303, 0x00304, 0x00305, 0x00306, 0x00307,
     0x00308, 0x00309, 0x0030a, 0x0030b, 0x0030c, 0x0030d, 0x0030e, 0x0030f,
     0x00310, 0x00311, 0x00312, 0x00313, 0x00314, 0x00315, 0x00316, 0x00317,
@@ -149,7 +163,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_007[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_007[128]
+    nxt_aligned(64) =
+{
     0x00380, 0x00381, 0x00382, 0x00383, 0x00384, 0x00385, 0x00386, 0x00387,
     0x00388, 0x00389, 0x0038a, 0x0038b, 0x0038c, 0x0038d, 0x0038e, 0x0038f,
     0x00390, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397,
@@ -169,7 +185,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_008[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_008[128]
+    nxt_aligned(64) =
+{
     0x00400, 0x00401, 0x00402, 0x00403, 0x00404, 0x00405, 0x00406, 0x00407,
     0x00408, 0x00409, 0x0040a, 0x0040b, 0x0040c, 0x0040d, 0x0040e, 0x0040f,
     0x00410, 0x00411, 0x00412, 0x00413, 0x00414, 0x00415, 0x00416, 0x00417,
@@ -189,7 +207,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_009[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_009[128]
+    nxt_aligned(64) =
+{
     0x00480, 0x00480, 0x00482, 0x00483, 0x00484, 0x00485, 0x00486, 0x00487,
     0x00488, 0x00489, 0x0048a, 0x0048a, 0x0048c, 0x0048c, 0x0048e, 0x0048e,
     0x00490, 0x00490, 0x00492, 0x00492, 0x00494, 0x00494, 0x00496, 0x00496,
@@ -209,7 +229,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_00a[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_00a[128]
+    nxt_aligned(64) =
+{
     0x00500, 0x00500, 0x00502, 0x00502, 0x00504, 0x00504, 0x00506, 0x00506,
     0x00508, 0x00508, 0x0050a, 0x0050a, 0x0050c, 0x0050c, 0x0050e, 0x0050e,
     0x00510, 0x00510, 0x00512, 0x00512, 0x00514, 0x00514, 0x00516, 0x00516,
@@ -229,7 +251,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_00b[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_00b[128]
+    nxt_aligned(64) =
+{
     0x00550, 0x00551, 0x00552, 0x00553, 0x00554, 0x00555, 0x00556, 0x00587,
     0x00588, 0x00589, 0x0058a, 0x0058b, 0x0058c, 0x0058d, 0x0058e, 0x0058f,
     0x00590, 0x00591, 0x00592, 0x00593, 0x00594, 0x00595, 0x00596, 0x00597,
@@ -249,7 +273,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_03a[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_03a[128]
+    nxt_aligned(64) =
+{
     0x01d00, 0x01d01, 0x01d02, 0x01d03, 0x01d04, 0x01d05, 0x01d06, 0x01d07,
     0x01d08, 0x01d09, 0x01d0a, 0x01d0b, 0x01d0c, 0x01d0d, 0x01d0e, 0x01d0f,
     0x01d10, 0x01d11, 0x01d12, 0x01d13, 0x01d14, 0x01d15, 0x01d16, 0x01d17,
@@ -269,7 +295,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_03c[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_03c[128]
+    nxt_aligned(64) =
+{
     0x01e00, 0x01e00, 0x01e02, 0x01e02, 0x01e04, 0x01e04, 0x01e06, 0x01e06,
     0x01e08, 0x01e08, 0x01e0a, 0x01e0a, 0x01e0c, 0x01e0c, 0x01e0e, 0x01e0e,
     0x01e10, 0x01e10, 0x01e12, 0x01e12, 0x01e14, 0x01e14, 0x01e16, 0x01e16,
@@ -289,7 +317,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_03d[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_03d[128]
+    nxt_aligned(64) =
+{
     0x01e80, 0x01e80, 0x01e82, 0x01e82, 0x01e84, 0x01e84, 0x01e86, 0x01e86,
     0x01e88, 0x01e88, 0x01e8a, 0x01e8a, 0x01e8c, 0x01e8c, 0x01e8e, 0x01e8e,
     0x01e90, 0x01e90, 0x01e92, 0x01e92, 0x01e94, 0x01e94, 0x01e96, 0x01e97,
@@ -309,7 +339,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_03e[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_03e[128]
+    nxt_aligned(64) =
+{
     0x01f08, 0x01f09, 0x01f0a, 0x01f0b, 0x01f0c, 0x01f0d, 0x01f0e, 0x01f0f,
     0x01f08, 0x01f09, 0x01f0a, 0x01f0b, 0x01f0c, 0x01f0d, 0x01f0e, 0x01f0f,
     0x01f18, 0x01f19, 0x01f1a, 0x01f1b, 0x01f1c, 0x01f1d, 0x01f16, 0x01f17,
@@ -329,7 +361,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_03f[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_03f[128]
+    nxt_aligned(64) =
+{
     0x01f88, 0x01f89, 0x01f8a, 0x01f8b, 0x01f8c, 0x01f8d, 0x01f8e, 0x01f8f,
     0x01f88, 0x01f89, 0x01f8a, 0x01f8b, 0x01f8c, 0x01f8d, 0x01f8e, 0x01f8f,
     0x01f98, 0x01f99, 0x01f9a, 0x01f9b, 0x01f9c, 0x01f9d, 0x01f9e, 0x01f9f,
@@ -349,7 +383,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_042[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_042[128]
+    nxt_aligned(64) =
+{
     0x02100, 0x02101, 0x02102, 0x02103, 0x02104, 0x02105, 0x02106, 0x02107,
     0x02108, 0x02109, 0x0210a, 0x0210b, 0x0210c, 0x0210d, 0x0210e, 0x0210f,
     0x02110, 0x02111, 0x02112, 0x02113, 0x02114, 0x02115, 0x02116, 0x02117,
@@ -369,7 +405,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_043[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_043[128]
+    nxt_aligned(64) =
+{
     0x02180, 0x02181, 0x02182, 0x02183, 0x02183, 0x02185, 0x02186, 0x02187,
     0x02188, 0x02189, 0x0218a, 0x0218b, 0x0218c, 0x0218d, 0x0218e, 0x0218f,
     0x02190, 0x02191, 0x02192, 0x02193, 0x02194, 0x02195, 0x02196, 0x02197,
@@ -389,7 +427,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_049[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_049[128]
+    nxt_aligned(64) =
+{
     0x02480, 0x02481, 0x02482, 0x02483, 0x02484, 0x02485, 0x02486, 0x02487,
     0x02488, 0x02489, 0x0248a, 0x0248b, 0x0248c, 0x0248d, 0x0248e, 0x0248f,
     0x02490, 0x02491, 0x02492, 0x02493, 0x02494, 0x02495, 0x02496, 0x02497,
@@ -409,7 +449,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_058[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_058[128]
+    nxt_aligned(64) =
+{
     0x02c00, 0x02c01, 0x02c02, 0x02c03, 0x02c04, 0x02c05, 0x02c06, 0x02c07,
     0x02c08, 0x02c09, 0x02c0a, 0x02c0b, 0x02c0c, 0x02c0d, 0x02c0e, 0x02c0f,
     0x02c10, 0x02c11, 0x02c12, 0x02c13, 0x02c14, 0x02c15, 0x02c16, 0x02c17,
@@ -429,7 +471,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_059[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_059[128]
+    nxt_aligned(64) =
+{
     0x02c80, 0x02c80, 0x02c82, 0x02c82, 0x02c84, 0x02c84, 0x02c86, 0x02c86,
     0x02c88, 0x02c88, 0x02c8a, 0x02c8a, 0x02c8c, 0x02c8c, 0x02c8e, 0x02c8e,
     0x02c90, 0x02c90, 0x02c92, 0x02c92, 0x02c94, 0x02c94, 0x02c96, 0x02c96,
@@ -449,7 +493,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_05a[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_05a[128]
+    nxt_aligned(64) =
+{
     0x010a0, 0x010a1, 0x010a2, 0x010a3, 0x010a4, 0x010a5, 0x010a6, 0x010a7,
     0x010a8, 0x010a9, 0x010aa, 0x010ab, 0x010ac, 0x010ad, 0x010ae, 0x010af,
     0x010b0, 0x010b1, 0x010b2, 0x010b3, 0x010b4, 0x010b5, 0x010b6, 0x010b7,
@@ -469,7 +515,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_14c[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_14c[128]
+    nxt_aligned(64) =
+{
     0x0a600, 0x0a601, 0x0a602, 0x0a603, 0x0a604, 0x0a605, 0x0a606, 0x0a607,
     0x0a608, 0x0a609, 0x0a60a, 0x0a60b, 0x0a60c, 0x0a60d, 0x0a60e, 0x0a60f,
     0x0a610, 0x0a611, 0x0a612, 0x0a613, 0x0a614, 0x0a615, 0x0a616, 0x0a617,
@@ -489,7 +537,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_14d[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_14d[128]
+    nxt_aligned(64) =
+{
     0x0a680, 0x0a680, 0x0a682, 0x0a682, 0x0a684, 0x0a684, 0x0a686, 0x0a686,
     0x0a688, 0x0a688, 0x0a68a, 0x0a68a, 0x0a68c, 0x0a68c, 0x0a68e, 0x0a68e,
     0x0a690, 0x0a690, 0x0a692, 0x0a692, 0x0a694, 0x0a694, 0x0a696, 0x0a696,
@@ -509,7 +559,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_14e[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_14e[128]
+    nxt_aligned(64) =
+{
     0x0a700, 0x0a701, 0x0a702, 0x0a703, 0x0a704, 0x0a705, 0x0a706, 0x0a707,
     0x0a708, 0x0a709, 0x0a70a, 0x0a70b, 0x0a70c, 0x0a70d, 0x0a70e, 0x0a70f,
     0x0a710, 0x0a711, 0x0a712, 0x0a713, 0x0a714, 0x0a715, 0x0a716, 0x0a717,
@@ -529,7 +581,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_14f[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_14f[128]
+    nxt_aligned(64) =
+{
     0x0a780, 0x0a780, 0x0a782, 0x0a782, 0x0a784, 0x0a784, 0x0a786, 0x0a786,
     0x0a788, 0x0a789, 0x0a78a, 0x0a78b, 0x0a78b, 0x0a78d, 0x0a78e, 0x0a78f,
     0x0a790, 0x0a790, 0x0a792, 0x0a792, 0x0a794, 0x0a795, 0x0a796, 0x0a797,
@@ -549,7 +603,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_1fe[128]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_1fe[128]
+    nxt_aligned(64) =
+{
     0x0ff00, 0x0ff01, 0x0ff02, 0x0ff03, 0x0ff04, 0x0ff05, 0x0ff06, 0x0ff07,
     0x0ff08, 0x0ff09, 0x0ff0a, 0x0ff0b, 0x0ff0c, 0x0ff0d, 0x0ff0e, 0x0ff0f,
     0x0ff10, 0x0ff11, 0x0ff12, 0x0ff13, 0x0ff14, 0x0ff15, 0x0ff16, 0x0ff17,
@@ -569,7 +625,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  nxt_unicode_upper_case_block_208[80]  nxt_aligned(64) = {
+static const uint32_t  nxt_unicode_upper_case_block_208[80]
+    nxt_aligned(64) =
+{
     0x10400, 0x10401, 0x10402, 0x10403, 0x10404, 0x10405, 0x10406, 0x10407,
     0x10408, 0x10409, 0x1040a, 0x1040b, 0x1040c, 0x1040d, 0x1040e, 0x1040f,
     0x10410, 0x10411, 0x10412, 0x10413, 0x10414, 0x10415, 0x10416, 0x10417,
@@ -583,7 +641,9 @@ static const uint32_t  nxt_unicode_upper
 };
 
 
-static const uint32_t  *nxt_unicode_upper_case_blocks[]  nxt_aligned(64) = {
+static const uint32_t  *nxt_unicode_upper_case_blocks[]
+    nxt_aligned(64) =
+{
     nxt_unicode_upper_case_block_000,
     nxt_unicode_upper_case_block_001,
     nxt_unicode_upper_case_block_002,
diff -r bc306c37d037 -r 49a8c7cfb121 nxt/nxt_unicode_upper_case.pl
--- a/nxt/nxt_unicode_upper_case.pl	Tue Aug 08 17:47:55 2017 +0300
+++ b/nxt/nxt_unicode_upper_case.pl	Tue Aug 08 17:47:56 2017 +0300
@@ -56,7 +56,9 @@ for my $block (sort { $a <=> $b } keys %
     my $block_size = ($block != $max_block) ? BLOCK_SIZE : $last_block_size;
 
     print "static const uint32_t  ";
-    printf("nxt_unicode_upper_case_block_%03x[%d]  nxt_aligned(64) = {",
+    printf("nxt_unicode_upper_case_block_%03x[%d]\n" .
+           "    nxt_aligned(64) =\n" .
+           "{",
            $block, $block_size);
 
     for my $c (0 .. $block_size - 1) {
@@ -77,7 +79,9 @@ for my $block (sort { $a <=> $b } keys %
 }
 
 
-print "static const uint32_t  *nxt_unicode_upper_case_blocks[]  nxt_aligned(64) = {\n";
+print "static const uint32_t  *nxt_unicode_upper_case_blocks[]\n" .
+      "    nxt_aligned(64) =\n" .
+      "{\n";
 
 for my $block (0 .. $max_block) {
     if (exists($blocks{$block})) {

From igor at sysoev.ru  Tue Aug  8 15:07:30 2017
From: igor at sysoev.ru (Igor Sysoev)
Date: Tue, 08 Aug 2017 15:07:30 +0000
Subject: [njs] Disabled njs_object_prop_alloc() inlining.
Message-ID: <hg.bd7e5e51b6be.1502204850.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/bd7e5e51b6be
branches:  
changeset: 394:bd7e5e51b6be
user:      Igor Sysoev <igor at sysoev.ru>
date:      Tue Aug 08 18:05:11 2017 +0300
description:
Disabled njs_object_prop_alloc() inlining.

diffstat:

 njs/njs_object.c |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (12 lines):

diff -r 49a8c7cfb121 -r bd7e5e51b6be njs/njs_object.c
--- a/njs/njs_object.c	Tue Aug 08 17:47:56 2017 +0300
+++ b/njs/njs_object.c	Tue Aug 08 18:05:11 2017 +0300
@@ -173,7 +173,7 @@ njs_object_hash_test(nxt_lvlhsh_query_t 
 }
 
 
-njs_object_prop_t *
+nxt_noinline njs_object_prop_t *
 njs_object_prop_alloc(njs_vm_t *vm, const njs_value_t *name,
     const njs_value_t *value, uint8_t attributes)
 {

From igor at sysoev.ru  Tue Aug  8 15:07:31 2017
From: igor at sysoev.ru (Igor Sysoev)
Date: Tue, 08 Aug 2017 15:07:31 +0000
Subject: [njs] Version 0.1.12.
Message-ID: <hg.c07b060396be.1502204851.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/c07b060396be
branches:  
changeset: 395:c07b060396be
user:      Igor Sysoev <igor at sysoev.ru>
date:      Tue Aug 08 18:05:14 2017 +0300
description:
Version 0.1.12.

diffstat:

 CHANGES  |  6 ++++++
 Makefile |  2 +-
 2 files changed, 7 insertions(+), 1 deletions(-)

diffs (23 lines):

diff -r bd7e5e51b6be -r c07b060396be CHANGES
--- a/CHANGES	Tue Aug 08 18:05:11 2017 +0300
+++ b/CHANGES	Tue Aug 08 18:05:14 2017 +0300
@@ -1,3 +1,9 @@
+
+Changes with nJScript 0.1.12                                     08 Aug 2017
+
+    *) Feature: Interactive shell.
+
+    *) Bugfix: in Object.isSealed().
 
 Changes with nJScript 0.1.11                                     27 Jun 2017
 
diff -r bd7e5e51b6be -r c07b060396be Makefile
--- a/Makefile	Tue Aug 08 18:05:11 2017 +0300
+++ b/Makefile	Tue Aug 08 18:05:14 2017 +0300
@@ -1,5 +1,5 @@
 
-NJS_VER =	0.1.11
+NJS_VER =	0.1.12
 
 NXT_LIB =	nxt
 

From igor at sysoev.ru  Tue Aug  8 15:07:33 2017
From: igor at sysoev.ru (Igor Sysoev)
Date: Tue, 08 Aug 2017 15:07:33 +0000
Subject: [njs] Added tag 0.1.12 for changeset c07b060396be
Message-ID: <hg.f47b1eed6ed8.1502204853.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/f47b1eed6ed8
branches:  
changeset: 396:f47b1eed6ed8
user:      Igor Sysoev <igor at sysoev.ru>
date:      Tue Aug 08 18:06:53 2017 +0300
description:
Added tag 0.1.12 for changeset c07b060396be

diffstat:

 .hgtags |  1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diffs (8 lines):

diff -r c07b060396be -r f47b1eed6ed8 .hgtags
--- a/.hgtags	Tue Aug 08 18:05:14 2017 +0300
+++ b/.hgtags	Tue Aug 08 18:06:53 2017 +0300
@@ -10,3 +10,4 @@ a29f29d481125db6101ecdc23dc20187c143cdc9
 5bd2833988222900f60ad9b330ebc44df3b30662 0.1.9
 b1456ef3e002376d9d146a8a02acf6a4a21748e9 0.1.10
 fc5df33f4e6b02a673daf3728ff690fb1e09b95e 0.1.11
+c07b060396be3622ca97b037a86076b61b850847 0.1.12

From mdounin at mdounin.ru  Tue Aug  8 18:03:56 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 8 Aug 2017 21:03:56 +0300
Subject: [PATCH 12 of 14] Proxy: add HTTP/2 support
In-Reply-To: <CAF-CG++xPsM1Ry4_=WsOnrqiTUM0uLs5BgDKZOZWsX91=n8HEg@mail.gmail.com>
References: <5f5d70428655db0889a2.1498163585@piotrsikora.sfo.corp.google.com>
 <7eb807b056da7abe9c67.1498163596@piotrsikora.sfo.corp.google.com>
 <20170719143430.GO93611@mdounin.ru>
 <CAF-CG++xPsM1Ry4_=WsOnrqiTUM0uLs5BgDKZOZWsX91=n8HEg@mail.gmail.com>
Message-ID: <20170808180356.GV93611@mdounin.ru>

Hello!

On Tue, Jul 25, 2017 at 06:28:16PM -0700, Piotr Sikora wrote:

> Hey Maxim,
> 
> > There are serious concerns about this and with other patches.
> > Most generic ones are outlined below.
> >
> > 1. Server-side HTTP/2 code complexity concerns.
> >
> > As per discussion with Valentin, introducing client-related code
> > paths in the server HTTP/2 code seems to complicate things.  Given
> > that the complexity of the code is already high, it might be
> > better idea to implement separate client-side processing instead.
> 
> I strongly disagree. Changes introduced to ngx_http_v2.c and
> ngx_http_v2_filter_module.c are minimal.
> 
> Also, having separate client-side and server-side code would result in
> thousands of lines of duplicated code, for no reason, really.

Biggest concert here are various "if (h2c->server) { ... } else { 
... }" constructions.  Certainly there can (and probably should) 
be some degree of code reuse between client and server code, 
though it would be much better to keep them easily separateable.

> > 2. Different protocols in proxy module.
> >
> > You've introduced a completely different protocol in the proxy
> > module, which contradicts the established practice of using
> > separate modules for different protocols.
> >
> > Reasons for having all (or at least may of) the protocols
> > available in the proxy module instead are well understood, and
> > there is a long-term plan to revise current practice.  The plan is
> > to preserve protocol-specific modules as a separate entities, but
> > let them share the common configuration directives, similar to how
> > it is currently done with upstream{} blocks and the overall
> > upstream infrastructure.  So one can write something like
> >
> >     proxy_pass fastcgi://127.0.0.1:9000;
> >
> > and get an expected result.
> >
> > While benefits of having all protocols sharing the same
> > user-visible interface are understood, approach taken with HTTP/2
> > implementation is considered suboptimal, and will likely lead to
> > something hard to maintain.
> >
> > I see two possible solutions here:
> >
> > - make HTTP/2-to-upstreams a separate full-featured upstream-based
> >   module, similar to proxy or fastcgi;
> 
> But that's basically:
> 
>     cat ngx_http_proxy_module.c \
>         | sed 's/ngx_http_proxy/ngx_http_proxy_v2/g' \
>         > ngx_http_proxy_v2_module.c
> 
> I don't see how copying ~4500 lines of code is a good idea.

This is how our upstream protocol modules are currently 
implemented: each module provides a configuration-specific parts, 
as well as it's protocol-specific parts.  Reducing duplication of 
configuration parsing is one of the main reasons of the "all 
protocols in the proxy module" plan as described above.  This is 
not what we currently have though, and introducing ad-hock hacks 
to save some lines of code looks much worse than some amount of 
code duplication.

Also, keeping the module separate would actually allow to reduce 
amount of code needed in some cases.  For example, since there is no 
cache support, at least in initial implementation, it would be 
possible to simply omit relevant directives.

> Also, as mentioned elsewhere, one of the reasons for adding HTTP/2
> code to the proxy module was the ability to negotiate HTTP/1.1 or
> HTTP/2 via ALPN. Creating a separate HTTP/2-to-upstreams module won't
> allow to add such feature in the future.

Such feature is anyway does not look compatible with current 
upstream infrastructure, as request is created once, before even 
connecting to backend servers, and it is not possible to switch to 
a different protocol after a connection.

> > - start working on the different protocols in the proxy module,
> >   and introduce HTTP/2 proxying after this work is done.
> 
> How is that even an option here? It's going to take forever before
> such rewrite is done, and I have no desire nor time to work on that.

Well, if you insist on doing things in the proxy module, posponing 
HTTP/2 till the rewrite is done is probably the only option.  We 
can consider prioritizing the rewrite rewrite accordingly.

[...]

> > - Upstream infrastructure as available in
> >   src/http/ngx_http_upstream.c is expected to be
> >   protocol-agnostic.  Introducing calls like
> >   ngx_http_v2_init_connection() there is a layering violation.
> >   Instead, there should be something more generic.
> 
> I agree that ngx_http_v2_init_connection() isn't perfect, however,
> fake connections are an abstraction layer that needs to be added
> somewhere. The same is done for SSL, and it's somehow acceptable.
> 
> I'm happy to use whatever generic mechanism is available, but there is
> none at the moment, and I don't see how adding even more code and
> complexity to this already big patchset is going to get us anywhere.

The problem here is that HTTP/2 is not a separate layer, in 
contrast to SSL, but rather tries to be an application-level 
protocol.  On the other hand, this is probably the problem of 
HTTP/2 protocol itself, and we have to deal with it somehow.

> > - Doing protocol parsing elsewhere instead of parsing things based
> >   on what nginx got from the connection results in some generic
> >   upstream mechanisms rendered not working - notably, it is no
> >   longer possible to simply write headers to a cache file.
> >   Serialization introduced instead, at least in its current form,
> >   looks more like a bandaid.
> 
> Except that HTTP/2 headers as read from the wire, even if parsed in
> separate module, couldn't be simply written to a cache file, because
> of stateful HPACK encoding, so serialization would need to be done
> anyway.

When needed, it may better idea to keep serialization within the 
HTTP/2 protocol module instead, similarly to how we skip large 
stderr in FastCGI.

> Anyway, it appears that your "serious concerns" are mostly about
> organization of the code, and not the code itself. What's unclear to
> me is how much of the code review this patchset actually received,
> i.e. if the existing code would be moved to a separate
> HTTP/2-to-upstreams module, would it be acceptable or do you have
> other issues with the code?

As of now, the code doesn't not look committable even from 
architecture point of view.  It is yet to recieve further review 
if/when architectural issues are resolved.

-- 
Maxim Dounin
http://nginx.org/

From mdounin at mdounin.ru  Tue Aug  8 19:25:19 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 8 Aug 2017 22:25:19 +0300
Subject: [PATCH] Add proxy_protocol option to mail listener
In-Reply-To: <1502112192.14660.18.camel@gmail.com>
References: <1502112192.14660.18.camel@gmail.com>
Message-ID: <20170808192518.GY93611@mdounin.ru>

Hello!

On Mon, Aug 07, 2017 at 03:23:12PM +0200, Kees Bos wrote:

> # HG changeset patch
> # User Kees Bos <cornelis.bos at gmail.com>
> # Date 1500565189 0
> #??????Thu Jul 20 15:39:49 2017 +0000
> # Node ID 327f18e079b175b14277a23e75715f5feee34d69
> # Parent??863b862534d7ac0dbf8babf68b824de6fb0d6ef4
> Add proxy_protocol option to mail listener

Style, should be:

Mail: added proxy_protocol option to mail listener.

> 
> Add support for the mail handlers. This enables the use of an upstream
> loadbalancer/proxy that connects with the proxy protocol. Examples of
> this are haproxy or a nginx stream handler that uses then proxy protocol
> in client conections.
> 
> The proxy protocol source ip address will we exposed to the auth
> handler as 'Proxy-Protocol-IP'.

Probably this should be a separate patch.  That is, one patch for 
"listen ... proxy_protocol" and "Proxy-Protocol-IP", and another 
one for set_real_ip_from (see below).

> 
> If the sender ip address matches one or more "set_real_ip_from" directives,
> the source ip address as specified in the in the proxy protocol will be
> used as 'Client-IP' in the authentication call and as address in the
> XCLIENT call.
> 
> Example config:
> mail {
> ????server_name mail.example.com;
> ????auth_http???localhost:9000/;
> 
> ????server {
> ????????listen 143 proxy_protocol;
> ????????protocol imap;
> ????}
> 
> ????server {
> ????????listen 25 proxy_protocol;
> ????????protocol smtp;
> ????????set_real_ip_from 127.0.0.0/8;
> ????????set_real_ip_from ::/128;
> ????}
> }
> 
> In the imap config, the source address given in the proxy protocol will
> never be used as Client-IP.
> 
> In the smtp config, the source address given in the proxy protocol will
> only be used as XCLIENT address when the sender address matches the
> "set_real_ip_from" settings (in this case only loopback address).

This doesn't seem to match what is expected to happen whan a real 
IP is set.

Much like in http and stream modules, if set_real_ip_from includes 
the client's address, then the client's address have to be 
replaced with one provided via PROXY protocol.  And then this 
address will be used everywhere where nginx uses client's address.

It looks like the set_real_ip_from part might not be familiar to 
you, so please consider preserving only the first patch, with 
"listen ... proxy_protocol" and "Proxy-Protocol-IP".

[...]

-- 
Maxim Dounin
http://nginx.org/

From pluknet at nginx.com  Wed Aug  9 12:05:36 2017
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Wed, 09 Aug 2017 12:05:36 +0000
Subject: [nginx] Version bump.
Message-ID: <hg.2284008a2791.1502280336.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/2284008a2791
branches:  
changeset: 7085:2284008a2791
user:      Sergey Kandaurov <pluknet at nginx.com>
date:      Wed Aug 09 14:59:44 2017 +0300
description:
Version bump.

diffstat:

 src/core/nginx.h |  4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diffs (14 lines):

diff -r 2f48ab272052 -r 2284008a2791 src/core/nginx.h
--- a/src/core/nginx.h	Tue Aug 08 18:00:11 2017 +0300
+++ b/src/core/nginx.h	Wed Aug 09 14:59:44 2017 +0300
@@ -9,8 +9,8 @@
 #define _NGINX_H_INCLUDED_
 
 
-#define nginx_version      1013004
-#define NGINX_VERSION      "1.13.4"
+#define nginx_version      1013005
+#define NGINX_VERSION      "1.13.5"
 #define NGINX_VER          "nginx/" NGINX_VERSION
 
 #ifdef NGX_BUILD

From pluknet at nginx.com  Wed Aug  9 12:05:40 2017
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Wed, 09 Aug 2017 12:05:40 +0000
Subject: [nginx] Style.
Message-ID: <hg.577628e6b6a6.1502280340.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/577628e6b6a6
branches:  
changeset: 7086:577628e6b6a6
user:      Sergey Kandaurov <pluknet at nginx.com>
date:      Wed Aug 09 14:59:46 2017 +0300
description:
Style.

diffstat:

 src/core/ngx_conf_file.c               |  1 +
 src/event/ngx_event_openssl.c          |  2 ++
 src/http/modules/ngx_http_geo_module.c |  1 +
 src/os/unix/ngx_files.c                |  1 +
 src/os/win32/ngx_files.c               |  1 +
 src/stream/ngx_stream_geo_module.c     |  1 +
 6 files changed, 7 insertions(+), 0 deletions(-)

diffs (74 lines):

diff -r 2284008a2791 -r 577628e6b6a6 src/core/ngx_conf_file.c
--- a/src/core/ngx_conf_file.c	Wed Aug 09 14:59:44 2017 +0300
+++ b/src/core/ngx_conf_file.c	Wed Aug 09 14:59:46 2017 +0300
@@ -178,6 +178,7 @@ ngx_conf_parse(ngx_conf_t *cf, ngx_str_t
         /* open configuration file */
 
         fd = ngx_open_file(filename->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
+
         if (fd == NGX_INVALID_FILE) {
             ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno,
                                ngx_open_file_n " \"%s\" failed",
diff -r 2284008a2791 -r 577628e6b6a6 src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c	Wed Aug 09 14:59:44 2017 +0300
+++ b/src/event/ngx_event_openssl.c	Wed Aug 09 14:59:46 2017 +0300
@@ -924,6 +924,7 @@ ngx_ssl_read_password_file(ngx_conf_t *c
     cln->data = passwords;
 
     fd = ngx_open_file(file->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
+
     if (fd == NGX_INVALID_FILE) {
         ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno,
                            ngx_open_file_n " \"%s\" failed", file->data);
@@ -2906,6 +2907,7 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *
         file.log = cf->log;
 
         file.fd = ngx_open_file(file.name.data, NGX_FILE_RDONLY, 0, 0);
+
         if (file.fd == NGX_INVALID_FILE) {
             ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno,
                                ngx_open_file_n " \"%V\" failed", &file.name);
diff -r 2284008a2791 -r 577628e6b6a6 src/http/modules/ngx_http_geo_module.c
--- a/src/http/modules/ngx_http_geo_module.c	Wed Aug 09 14:59:44 2017 +0300
+++ b/src/http/modules/ngx_http_geo_module.c	Wed Aug 09 14:59:46 2017 +0300
@@ -1401,6 +1401,7 @@ ngx_http_geo_include_binary_base(ngx_con
     file.log = cf->log;
 
     file.fd = ngx_open_file(name->data, NGX_FILE_RDONLY, 0, 0);
+
     if (file.fd == NGX_INVALID_FILE) {
         err = ngx_errno;
         if (err != NGX_ENOENT) {
diff -r 2284008a2791 -r 577628e6b6a6 src/os/unix/ngx_files.c
--- a/src/os/unix/ngx_files.c	Wed Aug 09 14:59:44 2017 +0300
+++ b/src/os/unix/ngx_files.c	Wed Aug 09 14:59:46 2017 +0300
@@ -620,6 +620,7 @@ ngx_create_file_mapping(ngx_file_mapping
 {
     fm->fd = ngx_open_file(fm->name, NGX_FILE_RDWR, NGX_FILE_TRUNCATE,
                            NGX_FILE_DEFAULT_ACCESS);
+
     if (fm->fd == NGX_INVALID_FILE) {
         ngx_log_error(NGX_LOG_CRIT, fm->log, ngx_errno,
                       ngx_open_file_n " \"%s\" failed", fm->name);
diff -r 2284008a2791 -r 577628e6b6a6 src/os/win32/ngx_files.c
--- a/src/os/win32/ngx_files.c	Wed Aug 09 14:59:44 2017 +0300
+++ b/src/os/win32/ngx_files.c	Wed Aug 09 14:59:46 2017 +0300
@@ -330,6 +330,7 @@ ngx_create_file_mapping(ngx_file_mapping
 
     fm->fd = ngx_open_file(fm->name, NGX_FILE_RDWR, NGX_FILE_TRUNCATE,
                            NGX_FILE_DEFAULT_ACCESS);
+
     if (fm->fd == NGX_INVALID_FILE) {
         ngx_log_error(NGX_LOG_CRIT, fm->log, ngx_errno,
                       ngx_open_file_n " \"%s\" failed", fm->name);
diff -r 2284008a2791 -r 577628e6b6a6 src/stream/ngx_stream_geo_module.c
--- a/src/stream/ngx_stream_geo_module.c	Wed Aug 09 14:59:44 2017 +0300
+++ b/src/stream/ngx_stream_geo_module.c	Wed Aug 09 14:59:46 2017 +0300
@@ -1327,6 +1327,7 @@ ngx_stream_geo_include_binary_base(ngx_c
     file.log = cf->log;
 
     file.fd = ngx_open_file(name->data, NGX_FILE_RDONLY, 0, 0);
+
     if (file.fd == NGX_INVALID_FILE) {
         err = ngx_errno;
         if (err != NGX_ENOENT) {

From pluknet at nginx.com  Wed Aug  9 12:05:42 2017
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Wed, 09 Aug 2017 12:05:42 +0000
Subject: [nginx] Fixed calls to ngx_open_file() in certain places.
Message-ID: <hg.47b7ffc3339d.1502280342.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/47b7ffc3339d
branches:  
changeset: 7087:47b7ffc3339d
user:      Sergey Kandaurov <pluknet at nginx.com>
date:      Wed Aug 09 15:03:27 2017 +0300
description:
Fixed calls to ngx_open_file() in certain places.

Pass NGX_FILE_OPEN to ngx_open_file() to fix "The parameter is incorrect"
error on win32 when using the ssl_session_ticket_key directive or loading
a binary geo base.  On UNIX, this change is a no-op.

diffstat:

 src/event/ngx_event_openssl.c          |  3 ++-
 src/http/modules/ngx_http_geo_module.c |  2 +-
 src/stream/ngx_stream_geo_module.c     |  2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diffs (37 lines):

diff -r 577628e6b6a6 -r 47b7ffc3339d src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c	Wed Aug 09 14:59:46 2017 +0300
+++ b/src/event/ngx_event_openssl.c	Wed Aug 09 15:03:27 2017 +0300
@@ -2906,7 +2906,8 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *
         file.name = path[i];
         file.log = cf->log;
 
-        file.fd = ngx_open_file(file.name.data, NGX_FILE_RDONLY, 0, 0);
+        file.fd = ngx_open_file(file.name.data, NGX_FILE_RDONLY,
+                                NGX_FILE_OPEN, 0);
 
         if (file.fd == NGX_INVALID_FILE) {
             ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno,
diff -r 577628e6b6a6 -r 47b7ffc3339d src/http/modules/ngx_http_geo_module.c
--- a/src/http/modules/ngx_http_geo_module.c	Wed Aug 09 14:59:46 2017 +0300
+++ b/src/http/modules/ngx_http_geo_module.c	Wed Aug 09 15:03:27 2017 +0300
@@ -1400,7 +1400,7 @@ ngx_http_geo_include_binary_base(ngx_con
     file.name = *name;
     file.log = cf->log;
 
-    file.fd = ngx_open_file(name->data, NGX_FILE_RDONLY, 0, 0);
+    file.fd = ngx_open_file(name->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
 
     if (file.fd == NGX_INVALID_FILE) {
         err = ngx_errno;
diff -r 577628e6b6a6 -r 47b7ffc3339d src/stream/ngx_stream_geo_module.c
--- a/src/stream/ngx_stream_geo_module.c	Wed Aug 09 14:59:46 2017 +0300
+++ b/src/stream/ngx_stream_geo_module.c	Wed Aug 09 15:03:27 2017 +0300
@@ -1326,7 +1326,7 @@ ngx_stream_geo_include_binary_base(ngx_c
     file.name = *name;
     file.log = cf->log;
 
-    file.fd = ngx_open_file(name->data, NGX_FILE_RDONLY, 0, 0);
+    file.fd = ngx_open_file(name->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
 
     if (file.fd == NGX_INVALID_FILE) {
         err = ngx_errno;

From sorin.v.manole at gmail.com  Wed Aug  9 20:17:13 2017
From: sorin.v.manole at gmail.com (Sorin Manole)
Date: Wed, 9 Aug 2017 23:17:13 +0300
Subject: set directive at http level
Message-ID: <CAH19K_ufHMCkFkkSjioEEuCup62haPjgEarvNDXwRVa5WnCxmg@mail.gmail.com>

Hello,

Do you think there is a problem in making the set directive available
at http level (right now it's only server and lower). This would allow
the usage of variables defined (and overridden in a lower config
level) by set in directives like "map" and "log_format".

Thanks.

From vbart at nginx.com  Wed Aug  9 21:03:23 2017
From: vbart at nginx.com (Valentin V. Bartenev)
Date: Thu, 10 Aug 2017 00:03:23 +0300
Subject: set directive at http level
In-Reply-To: <CAH19K_ufHMCkFkkSjioEEuCup62haPjgEarvNDXwRVa5WnCxmg@mail.gmail.com>
References: <CAH19K_ufHMCkFkkSjioEEuCup62haPjgEarvNDXwRVa5WnCxmg@mail.gmail.com>
Message-ID: <7513748.1fxqyeJJ0K@vbart-laptop>

On Wednesday 09 August 2017 23:17:13 Sorin Manole wrote:
> Hello,
> 
> Do you think there is a problem in making the set directive available
> at http level (right now it's only server and lower). This would allow
> the usage of variables defined (and overridden in a lower config
> level) by set in directives like "map" and "log_format".
> 
> Thanks.

1. The "set" directive is an instruction of the "rewrite" module.
   If you learn how it works, you will realize that "set" on the
   "http" level has no sense.

   Please, take a look at the "rewrite" module documentation:
   http://nginx.org/en/docs/http/ngx_http_rewrite_module.html

2. There's no problem in using variables defined by a "set" directive
   in "map" or "log_format" directives.  That's already allowed.


  wbr, Valentin V. Bartenev


From mdounin at mdounin.ru  Thu Aug 10 19:22:04 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 10 Aug 2017 19:22:04 +0000
Subject: [nginx] Style.
Message-ID: <hg.f684178faec9.1502392924.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/f684178faec9
branches:  
changeset: 7088:f684178faec9
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Thu Aug 10 22:21:20 2017 +0300
description:
Style.

diffstat:

 src/core/ngx_regex.c                     |  4 ++--
 src/http/modules/ngx_http_scgi_module.c  |  2 +-
 src/http/modules/ngx_http_uwsgi_module.c |  8 ++++----
 3 files changed, 7 insertions(+), 7 deletions(-)

diffs (71 lines):

diff --git a/src/core/ngx_regex.c b/src/core/ngx_regex.c
--- a/src/core/ngx_regex.c
+++ b/src/core/ngx_regex.c
@@ -262,7 +262,7 @@ ngx_pcre_free_studies(void *data)
     part = &studies->part;
     elts = part->elts;
 
-    for (i = 0 ; /* void */ ; i++) {
+    for (i = 0; /* void */ ; i++) {
 
         if (i >= part->nelts) {
             if (part->next == NULL) {
@@ -326,7 +326,7 @@ ngx_regex_module_init(ngx_cycle_t *cycle
     part = &ngx_pcre_studies->part;
     elts = part->elts;
 
-    for (i = 0 ; /* void */ ; i++) {
+    for (i = 0; /* void */ ; i++) {
 
         if (i >= part->nelts) {
             if (part->next == NULL) {
diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c
--- a/src/http/modules/ngx_http_scgi_module.c
+++ b/src/http/modules/ngx_http_scgi_module.c
@@ -819,7 +819,7 @@ ngx_http_scgi_create_request(ngx_http_re
             key = e.pos;
 #endif
             code = *(ngx_http_script_code_pt *) e.ip;
-            code((ngx_http_script_engine_t *) & e);
+            code((ngx_http_script_engine_t *) &e);
 
 #if (NGX_DEBUG)
             val = e.pos;
diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c
--- a/src/http/modules/ngx_http_uwsgi_module.c
+++ b/src/http/modules/ngx_http_uwsgi_module.c
@@ -865,7 +865,7 @@ ngx_http_uwsgi_create_request(ngx_http_r
             lcode = *(ngx_http_script_len_code_pt *) le.ip;
             skip_empty = lcode(&le);
 
-            for (val_len = 0; *(uintptr_t *) le.ip; val_len += lcode (&le)) {
+            for (val_len = 0; *(uintptr_t *) le.ip; val_len += lcode(&le)) {
                 lcode = *(ngx_http_script_len_code_pt *) le.ip;
             }
             le.ip += sizeof(uintptr_t);
@@ -990,7 +990,7 @@ ngx_http_uwsgi_create_request(ngx_http_r
         while (*(uintptr_t *) le.ip) {
 
             lcode = *(ngx_http_script_len_code_pt *) le.ip;
-            key_len = (u_char) lcode (&le);
+            key_len = (u_char) lcode(&le);
 
             lcode = *(ngx_http_script_len_code_pt *) le.ip;
             skip_empty = lcode(&le);
@@ -1018,14 +1018,14 @@ ngx_http_uwsgi_create_request(ngx_http_r
             *e.pos++ = (u_char) ((key_len >> 8) & 0xff);
 
             code = *(ngx_http_script_code_pt *) e.ip;
-            code((ngx_http_script_engine_t *) & e);
+            code((ngx_http_script_engine_t *) &e);
 
             *e.pos++ = (u_char) (val_len & 0xff);
             *e.pos++ = (u_char) ((val_len >> 8) & 0xff);
 
             while (*(uintptr_t *) e.ip) {
                 code = *(ngx_http_script_code_pt *) e.ip;
-                code((ngx_http_script_engine_t *) & e);
+                code((ngx_http_script_engine_t *) &e);
             }
 
             e.ip += sizeof(uintptr_t);

From mdounin at mdounin.ru  Thu Aug 10 19:22:13 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 10 Aug 2017 19:22:13 +0000
Subject: [nginx] Restored ngx_event_aio_t layout for debug logging.
Message-ID: <hg.371da6a7021b.1502392933.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/371da6a7021b
branches:  
changeset: 7089:371da6a7021b
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Thu Aug 10 22:21:22 2017 +0300
description:
Restored ngx_event_aio_t layout for debug logging.

The "fd" field should be after 3 pointers for ngx_event_ident() to use it.
This was broken by ccad84a174e0.  While it does not seem to be currently used
for aio-related events, it should be a good idea to preserve the correct
layout nevertheless.

diffstat:

 src/event/ngx_event.h |  4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diffs (18 lines):

diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h
--- a/src/event/ngx_event.h
+++ b/src/event/ngx_event.h
@@ -152,12 +152,12 @@ struct ngx_event_aio_s {
     ngx_event_handler_pt       handler;
     ngx_file_t                *file;
 
+    ngx_fd_t                   fd;
+
 #if (NGX_HAVE_AIO_SENDFILE || NGX_COMPAT)
     ssize_t                  (*preload_handler)(ngx_buf_t *file);
 #endif
 
-    ngx_fd_t                   fd;
-
 #if (NGX_HAVE_EVENTFD)
     int64_t                    res;
 #endif

From mdounin at mdounin.ru  Thu Aug 10 19:22:25 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 10 Aug 2017 19:22:25 +0000
Subject: [nginx] Range filter: changed type for total length to off_t.
Message-ID: <hg.a2f5e25d6a28.1502392945.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/a2f5e25d6a28
branches:  
changeset: 7090:a2f5e25d6a28
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Thu Aug 10 22:21:23 2017 +0300
description:
Range filter: changed type for total length to off_t.

Total length of a response with multiple ranges can be larger than a size_t
variable can hold, so type changed to off_t.  Previously, an incorrect
Content-Length was returned when requesting more than 4G of ranges from
a large enough file on a 32-bit system.

An additional size_t variable introduced to calculate size of the boundary
header buffer, as off_t is not needed here and will require type casts on
win32.

Reported by Shuxin Yang,
http://mailman.nginx.org/pipermail/nginx/2017-July/054384.html.

diffstat:

 src/http/modules/ngx_http_range_filter_module.c |  17 +++++++++--------
 1 files changed, 9 insertions(+), 8 deletions(-)

diffs (44 lines):

diff --git a/src/http/modules/ngx_http_range_filter_module.c b/src/http/modules/ngx_http_range_filter_module.c
--- a/src/http/modules/ngx_http_range_filter_module.c
+++ b/src/http/modules/ngx_http_range_filter_module.c
@@ -463,23 +463,24 @@ static ngx_int_t
 ngx_http_range_multipart_header(ngx_http_request_t *r,
     ngx_http_range_filter_ctx_t *ctx)
 {
-    size_t              len;
+    off_t               len;
+    size_t              size;
     ngx_uint_t          i;
     ngx_http_range_t   *range;
     ngx_atomic_uint_t   boundary;
 
-    len = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN
-          + sizeof(CRLF "Content-Type: ") - 1
-          + r->headers_out.content_type.len
-          + sizeof(CRLF "Content-Range: bytes ") - 1;
+    size = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN
+           + sizeof(CRLF "Content-Type: ") - 1
+           + r->headers_out.content_type.len
+           + sizeof(CRLF "Content-Range: bytes ") - 1;
 
     if (r->headers_out.content_type_len == r->headers_out.content_type.len
         && r->headers_out.charset.len)
     {
-        len += sizeof("; charset=") - 1 + r->headers_out.charset.len;
+        size += sizeof("; charset=") - 1 + r->headers_out.charset.len;
     }
 
-    ctx->boundary_header.data = ngx_pnalloc(r->pool, len);
+    ctx->boundary_header.data = ngx_pnalloc(r->pool, size);
     if (ctx->boundary_header.data == NULL) {
         return NGX_ERROR;
     }
@@ -569,7 +570,7 @@ ngx_http_range_multipart_header(ngx_http
                                      - range[i].content_range.data;
 
         len += ctx->boundary_header.len + range[i].content_range.len
-                                    + (size_t) (range[i].end - range[i].start);
+                                             + (range[i].end - range[i].start);
     }
 
     r->headers_out.content_length_n = len;

From pgnet.dev at gmail.com  Mon Aug 14 03:55:40 2017
From: pgnet.dev at gmail.com (PGNet Dev)
Date: Sun, 13 Aug 2017 20:55:40 -0700
Subject: [njs] Version 0.1.12. DOC BUG
In-Reply-To: <hg.c07b060396be.1502204851.5965299922797593991@dev.nginx.com>
References: <hg.c07b060396be.1502204851.5965299922797593991@dev.nginx.com>
Message-ID: <b79c90b7-409e-6758-436f-e2390d503a99@gmail.com>

fyi, site doc bug/typo

instructions @

	http://nginx.org/en/docs/njs_about.html

state

	...
	The modules can also be built as dynamic:

	    ./configure --add-dynamic_module=path-to-njs/nginx
	...

that's a typo.  should be,

	    ./configure --add-dynamic-module=path-to-njs/nginx


From eran.kornblau at kaltura.com  Mon Aug 14 10:12:02 2017
From: eran.kornblau at kaltura.com (Eran Kornblau)
Date: Mon, 14 Aug 2017 10:12:02 +0000
Subject: Add support for buffering in scripted logs
In-Reply-To: <AM4PR0401MB1731BFCCE414FD22A7FCF916F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
References: <AM4PR0401MB1731BFCCE414FD22A7FCF916F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
Message-ID: <AM4PR0401MB1731EBFC0FACAA933F2E0BC0F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>

Ping

Thanks

Eran

-----Original Message-----
From: nginx-devel [mailto:nginx-devel-bounces at nginx.org] On Behalf Of Eran Kornblau
Sent: Monday, August 7, 2017 2:09 PM
To: nginx-devel at nginx.org
Subject: RE: Add support for buffering in scripted logs

Sorry for spamming... forgot to write - something that I know is missing in this patch is support for 'reopen files' (USR1) In order to handle that, I probably need to add an rbtree on cycle->open_files (the O(n) search that is currently performed in ngx_conf_open_file doesn't seem appropriate for runtime) If the general concept in the below patch is approved, I will happily submit a patch for this as well.

Thanks

Eran

-----Original Message-----
From: Eran Kornblau
Sent: Monday, August 7, 2017 1:36 PM
To: nginx-devel at nginx.org
Subject: Add support for buffering is scripted logs

Hi all,

The attached patch adds support for log buffering when using variables in the access log file name.

The use case is this - we use nginx to receive analytics beacons and write them to the access log.
We'd like to have a log file per hour that contains the logs of the specific hour. If we use some external script to perform log rotate, we cannot avoid log lines slipping between adjacent files.
It's important for us to have the log lines partitioning accurate, in case we need to reindex a specific hour to the database.
Scripted logs seem perfect for this, but we don't want to give up on log compression, since the number of events can be high.

The patch relies on the open file cache to keep the context (buffer + flush event) of each file.
In order to do that, I added the ability to register callbacks in open file cache for these events:
1. init - new cached file object created 2. flush - the file handle of a cached file is closed 3. free - the cached file is destroyed The context is allocated right after the ngx_cached_open_file_t struct, this was done in order to avoid increasing the size of the ngx_cached_open_file_t struct (compared to the alternative of adding some opaque pointer on this struct).
Therefore, there are no memory implications for "regular" (log = 0) open file caches, the overhead is only an extra 'if' on the creation / deletion of the cached file object (no impact at all on cache hit, which is probably the most performance sensitive)

Btw, on a similar subject - can anyone explain the purpose of checking the existence of the root dir in scripted logs? Only explanation I could think of is to provide some security protection in case $uri is used in the script, but that sounds like a very specific use case to me...
Anyway, if that is indeed the case, maybe it makes sense to add conf directive to enable/disable this behavior?

Thank you!

Eran

_______________________________________________
nginx-devel mailing list
nginx-devel at nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel
-------------- next part --------------
A non-text attachment was scrubbed...
Name: buffered_script_logs.patch
Type: application/octet-stream
Size: 19571 bytes
Desc: buffered_script_logs.patch
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170814/8d8b9f74/attachment-0001.obj>

From pluknet at nginx.com  Mon Aug 14 10:49:34 2017
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Mon, 14 Aug 2017 13:49:34 +0300
Subject: [njs] Version 0.1.12. DOC BUG
In-Reply-To: <b79c90b7-409e-6758-436f-e2390d503a99@gmail.com>
References: <hg.c07b060396be.1502204851.5965299922797593991@dev.nginx.com>
 <b79c90b7-409e-6758-436f-e2390d503a99@gmail.com>
Message-ID: <A0970B27-56BB-46E8-9A58-C0D4992F1DA6@nginx.com>


> On 14 Aug 2017, at 06:55, PGNet Dev <pgnet.dev at gmail.com> wrote:
> 
> fyi, site doc bug/typo
> 
> instructions @
> 
> 	http://nginx.org/en/docs/njs_about.html
> 
> state
> 
> 	...
> 	The modules can also be built as dynamic:
> 
> 	    ./configure --add-dynamic_module=path-to-njs/nginx
> 	...
> 
> that's a typo.  should be,
> 
> 	    ./configure --add-dynamic-module=path-to-njs/nginx
> 

Fixed, thanks.

-- 
Sergey Kandaurov


From mdounin at mdounin.ru  Mon Aug 14 15:45:07 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 14 Aug 2017 18:45:07 +0300
Subject: Add support for buffering is scripted logs
In-Reply-To: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
References: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
Message-ID: <20170814154507.GV93611@mdounin.ru>

Hello!

On Mon, Aug 07, 2017 at 10:36:19AM +0000, Eran Kornblau wrote:

> Hi all,
> 
> The attached patch adds support for log buffering when using variables in the access log file name.
> 
> The use case is this - we use nginx to receive analytics beacons and write them to the access log.
> We'd like to have a log file per hour that contains the logs of the specific hour. If we use some external 
> script to perform log rotate, we cannot avoid log lines slipping between adjacent files.
> It's important for us to have the log lines partitioning accurate, in case we need to reindex a specific 
> hour to the database.
> Scripted logs seem perfect for this, but we don't want to give up on log compression, since the 
> number of events can be high.
> 
> The patch relies on the open file cache to keep the context (buffer + flush event) of each file.
> In order to do that, I added the ability to register callbacks in open file cache for these events:
> 1. init - new cached file object created
> 2. flush - the file handle of a cached file is closed
> 3. free - the cached file is destroyed
> The context is allocated right after the ngx_cached_open_file_t struct, this was done in order to avoid
> increasing the size of the ngx_cached_open_file_t struct (compared to the alternative of adding some
> opaque pointer on this struct).
> Therefore, there are no memory implications for "regular" (log = 0) open file caches, the overhead is
> only an extra 'if' on the creation / deletion of the cached file object (no impact at all on cache hit,
> which is probably the most performance sensitive) 

Just a quick note: for me, the whole feature looks questionable, 
and the implementation is far from being in a commitable state.

> Btw, on a similar subject - can anyone explain the purpose of checking the existence of the root dir
> in scripted logs? Only explanation I could think of is to provide some security protection in case $uri
> is used in the script, but that sounds like a very specific use case to me... 
> Anyway, if that is indeed the case, maybe it makes sense to add conf directive to enable/disable 
> this behavior?

The basic purpose of this restriction is to protect configurations 
like

    server {
        access_log /spool/vhost/logs/$host;
        ...
    }

from creating arbitrary log files, thus allowing DoS attacks via 
inode exhaustion.  Accordingly, the documentation 
(http://nginx.org/r/access_log) recommends to specify both root 
and access_log if you want to use variables in logging:

: during each log write the existence of the request?s root 
: directory is checked, and if it does not exist the log is not 
: created. It is thus a good idea to specify both root and 
: access_log on the same level:
:
: server {
:     root       /spool/vhost/data/$host;
:     access_log /spool/vhost/logs/$host;
:     ...

-- 
Maxim Dounin
http://nginx.org/

From eran.kornblau at kaltura.com  Mon Aug 14 16:01:10 2017
From: eran.kornblau at kaltura.com (Eran Kornblau)
Date: Mon, 14 Aug 2017 16:01:10 +0000
Subject: Add support for buffering is scripted logs
In-Reply-To: <20170814154507.GV93611@mdounin.ru>
References: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814154507.GV93611@mdounin.ru>
Message-ID: <AM4PR0401MB1731067E9B1437EA6184E779F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>

Thanks Maxim, comments inline.

> Hello!
> 
> On Mon, Aug 07, 2017 at 10:36:19AM +0000, Eran Kornblau wrote:
> 
> Just a quick note: for me, the whole feature looks questionable, and the implementation is far from being in a commitable state.
> 
Why questionable? you would probably agree that variables in the access log name can be useful in some cases,
and you'd probably also agree that log compression is good (it saves a lot of IO for a negligible amount of CPU -
we use it in all nginx servers we have). So why not allow both?
Regarding the implementation, I tried to match the coding standard (80 chars width, braces etc.) but looks like 
I missed... I will happily apply any changes you request.

> > Btw, on a similar subject - can anyone explain the purpose of checking 
> > the existence of the root dir in scripted logs? Only explanation I 
> > could think of is to provide some security protection in case $uri is used in the script, but that sounds like a very specific use case to me...
> > Anyway, if that is indeed the case, maybe it makes sense to add conf 
> > directive to enable/disable this behavior?
> 
> The basic purpose of this restriction is to protect configurations like
> 
>     server {
>         access_log /spool/vhost/logs/$host;
>         ...
>     }
> 
> from creating arbitrary log files, thus allowing DoS attacks via inode exhaustion.  Accordingly, the documentation
> (http://nginx.org/r/access_log) recommends to specify both root and access_log if you want to use variables in logging:
> 
> : during each log write the existence of the request?s root
> : directory is checked, and if it does not exist the log is not
> : created. It is thus a good idea to specify both root and
> : access_log on the same level:
> :
> : server {
> :     root       /spool/vhost/data/$host;
> :     access_log /spool/vhost/logs/$host;
> :     ...
> 
Thanks for the clarification, IMHO worth having it configurable (check_root=on/off), in my use case,
the variables do not depend on anything a client can control (there will be vars generated by the strftime 
module, and static strings from map), so the root check is simply irrelevant.
Will be happy to submit a patch for that later if you agree, though less a priority for me than this patch.

> --
> Maxim Dounin
> http://nginx.org/
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>

From mdounin at mdounin.ru  Mon Aug 14 16:54:44 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 14 Aug 2017 19:54:44 +0300
Subject: Add support for buffering is scripted logs
In-Reply-To: <AM4PR0401MB1731067E9B1437EA6184E779F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
References: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814154507.GV93611@mdounin.ru>
 <AM4PR0401MB1731067E9B1437EA6184E779F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
Message-ID: <20170814165444.GX93611@mdounin.ru>

Hello!

On Mon, Aug 14, 2017 at 04:01:10PM +0000, Eran Kornblau wrote:

> Thanks Maxim, comments inline.
> 
> > Hello!
> > 
> > On Mon, Aug 07, 2017 at 10:36:19AM +0000, Eran Kornblau wrote:
> > 
> > Just a quick note: for me, the whole feature looks questionable, and the implementation is far from being in a commitable state.
> > 
> Why questionable? you would probably agree that variables in the access log name can be useful in some cases,
> and you'd probably also agree that log compression is good (it saves a lot of IO for a negligible amount of CPU -
> we use it in all nginx servers we have). So why not allow both?

Obviously enough, every feature you can think of is useful in some 
cases.  The question is code complexity and other maintanance 
expenses compared to the usefulness of the particular feature.

[...]

> Thanks for the clarification, IMHO worth having it configurable (check_root=on/off), in my use case,
> the variables do not depend on anything a client can control (there will be vars generated by the strftime 
> module, and static strings from map), so the root check is simply irrelevant.
> Will be happy to submit a patch for that later if you agree, though less a priority for me than this patch.

No, thanks.

-- 
Maxim Dounin
http://nginx.org/

From eran.kornblau at kaltura.com  Mon Aug 14 17:11:22 2017
From: eran.kornblau at kaltura.com (Eran Kornblau)
Date: Mon, 14 Aug 2017 17:11:22 +0000
Subject: Add support for buffering is scripted logs
In-Reply-To: <20170814165444.GX93611@mdounin.ru>
References: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814154507.GV93611@mdounin.ru>
 <AM4PR0401MB1731067E9B1437EA6184E779F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814165444.GX93611@mdounin.ru>
Message-ID: <AM4PR0401MB173109E2D01BC0B95F72909EF58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>

> -----Original Message-----
> From: nginx-devel [mailto:nginx-devel-bounces at nginx.org] On Behalf Of Maxim Dounin
> Sent: Monday, August 14, 2017 7:55 PM
> To: nginx-devel at nginx.org
> Subject: Re: Add support for buffering is scripted logs
> 
> Hello!
> 
> On Mon, Aug 14, 2017 at 04:01:10PM +0000, Eran Kornblau wrote:
> 
> > Thanks Maxim, comments inline.
> > 
> > > Hello!
> > > 
> > > On Mon, Aug 07, 2017 at 10:36:19AM +0000, Eran Kornblau wrote:
> > > 
> > > Just a quick note: for me, the whole feature looks questionable, and the implementation is far from being in a commitable state.
> > > 
> > Why questionable? you would probably agree that variables in the 
> > access log name can be useful in some cases, and you'd probably also 
> > agree that log compression is good (it saves a lot of IO for a negligible amount of CPU - we use it in all nginx servers we have). So why not allow both?
> 
> Obviously enough, every feature you can think of is useful in some cases.  The question is code complexity and other maintanance expenses compared to the usefulness of the particular feature.
> 
Ok, so is that a final 'no' for this whole feature, or is there is anything else I can do to get this feature in?

Thanks

Eran

From mdounin at mdounin.ru  Mon Aug 14 17:33:46 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 14 Aug 2017 20:33:46 +0300
Subject: Add support for buffering is scripted logs
In-Reply-To: <AM4PR0401MB173109E2D01BC0B95F72909EF58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
References: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814154507.GV93611@mdounin.ru>
 <AM4PR0401MB1731067E9B1437EA6184E779F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814165444.GX93611@mdounin.ru>
 <AM4PR0401MB173109E2D01BC0B95F72909EF58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
Message-ID: <20170814173346.GY93611@mdounin.ru>

Hello!

On Mon, Aug 14, 2017 at 05:11:22PM +0000, Eran Kornblau wrote:

> > -----Original Message-----
> > From: nginx-devel [mailto:nginx-devel-bounces at nginx.org] On Behalf Of Maxim Dounin
> > Sent: Monday, August 14, 2017 7:55 PM
> > To: nginx-devel at nginx.org
> > Subject: Re: Add support for buffering is scripted logs
> > 
> > Hello!
> > 
> > On Mon, Aug 14, 2017 at 04:01:10PM +0000, Eran Kornblau wrote:
> > 
> > > Thanks Maxim, comments inline.
> > > 
> > > > Hello!
> > > > 
> > > > On Mon, Aug 07, 2017 at 10:36:19AM +0000, Eran Kornblau wrote:
> > > > 
> > > > Just a quick note: for me, the whole feature looks questionable, and the implementation is far from being in a commitable state.
> > > > 
> > > Why questionable? you would probably agree that variables in the 
> > > access log name can be useful in some cases, and you'd probably also 
> > > agree that log compression is good (it saves a lot of IO for a negligible amount of CPU - we use it in all nginx servers we have). So why not allow both?
> > 
> > Obviously enough, every feature you can think of is useful in some cases.  The question is code complexity and other maintanance expenses compared to the usefulness of the particular feature.
> > 
> Ok, so is that a final 'no' for this whole feature, or is there is anything else I can do to get this feature in?

It is certainly not a "final no".  As I wrote in the very first 
comment, a) it's just a quick note, nothing more, and b) the 
feature is questionable.  If a good implementation will be 
submitted, we can consider committing it.

-- 
Maxim Dounin
http://nginx.org/

From eran.kornblau at kaltura.com  Mon Aug 14 18:00:53 2017
From: eran.kornblau at kaltura.com (Eran Kornblau)
Date: Mon, 14 Aug 2017 18:00:53 +0000
Subject: Add support for buffering is scripted logs
In-Reply-To: <20170814173346.GY93611@mdounin.ru>
References: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814154507.GV93611@mdounin.ru>
 <AM4PR0401MB1731067E9B1437EA6184E779F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814165444.GX93611@mdounin.ru>
 <AM4PR0401MB173109E2D01BC0B95F72909EF58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814173346.GY93611@mdounin.ru>
Message-ID: <AM4PR0401MB17314B45CBA8DA7F3EA2DA13F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>

> 
> -----Original Message-----
> From: nginx-devel [mailto:nginx-devel-bounces at nginx.org] On Behalf Of Maxim Dounin
> Sent: Monday, August 14, 2017 8:34 PM
> To: nginx-devel at nginx.org
> Subject: Re: Add support for buffering is scripted logs
> 
> > Ok, so is that a final 'no' for this whole feature, or is there is anything else I can do to get this feature in?
> 
> It is certainly not a "final no".  As I wrote in the very first comment, a) it's just a quick note, nothing more, and b) the feature is questionable.  If a good implementation will be submitted, we can consider committing it.
> 
That's good, I thought you were just rejecting politely :)

It would be really great if you could point me to specific parts you think look bad.
For example, I'm guessing that you don't like the callbacks I added to open file cache,
but I was thinking that it's better to do it this way than to duplicate large chunks of code
and write an open file cache specific to log, please let me know if you think otherwise.
Any feedback you can provide will be appreciated

Thanks

Eran

> --
> Maxim Dounin
> http://nginx.org/
> _______________________________________________

From savetherbtz at gmail.com  Mon Aug 14 18:24:39 2017
From: savetherbtz at gmail.com (Alexey Ivanov)
Date: Mon, 14 Aug 2017 11:24:39 -0700
Subject: Add support for buffering is scripted logs
In-Reply-To: <AM4PR0401MB17314B45CBA8DA7F3EA2DA13F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
References: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814154507.GV93611@mdounin.ru>
 <AM4PR0401MB1731067E9B1437EA6184E779F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814165444.GX93611@mdounin.ru>
 <AM4PR0401MB173109E2D01BC0B95F72909EF58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814173346.GY93611@mdounin.ru>
 <AM4PR0401MB17314B45CBA8DA7F3EA2DA13F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
Message-ID: <7AEF0D22-0BAF-42E3-9DE8-248987CD4111@gmail.com>

using syslog for that particular usecase seem way more elegant, customizable, and simple. As a side bonus you won't block event loop on vfs operations (open/write/close).

> On Aug 14, 2017, at 11:00 AM, Eran Kornblau <eran.kornblau at kaltura.com> wrote:
> 
>> 
>> -----Original Message-----
>> From: nginx-devel [mailto:nginx-devel-bounces at nginx.org] On Behalf Of Maxim Dounin
>> Sent: Monday, August 14, 2017 8:34 PM
>> To: nginx-devel at nginx.org
>> Subject: Re: Add support for buffering is scripted logs
>> 
>>> Ok, so is that a final 'no' for this whole feature, or is there is anything else I can do to get this feature in?
>> 
>> It is certainly not a "final no".  As I wrote in the very first comment, a) it's just a quick note, nothing more, and b) the feature is questionable.  If a good implementation will be submitted, we can consider committing it.
>> 
> That's good, I thought you were just rejecting politely :)
> 
> It would be really great if you could point me to specific parts you think look bad.
> For example, I'm guessing that you don't like the callbacks I added to open file cache,
> but I was thinking that it's better to do it this way than to duplicate large chunks of code
> and write an open file cache specific to log, please let me know if you think otherwise.
> Any feedback you can provide will be appreciated
> 
> Thanks
> 
> Eran
> 
>> --
>> Maxim Dounin
>> http://nginx.org/
>> _______________________________________________
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 842 bytes
Desc: Message signed with OpenPGP
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170814/39f96930/attachment.bin>

From mdounin at mdounin.ru  Mon Aug 14 18:59:38 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 14 Aug 2017 21:59:38 +0300
Subject: Add support for buffering is scripted logs
In-Reply-To: <AM4PR0401MB17314B45CBA8DA7F3EA2DA13F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
References: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814154507.GV93611@mdounin.ru>
 <AM4PR0401MB1731067E9B1437EA6184E779F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814165444.GX93611@mdounin.ru>
 <AM4PR0401MB173109E2D01BC0B95F72909EF58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814173346.GY93611@mdounin.ru>
 <AM4PR0401MB17314B45CBA8DA7F3EA2DA13F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
Message-ID: <20170814185938.GZ93611@mdounin.ru>

Hello!

On Mon, Aug 14, 2017 at 06:00:53PM +0000, Eran Kornblau wrote:

> > 
> > -----Original Message-----
> > From: nginx-devel [mailto:nginx-devel-bounces at nginx.org] On Behalf Of Maxim Dounin
> > Sent: Monday, August 14, 2017 8:34 PM
> > To: nginx-devel at nginx.org
> > Subject: Re: Add support for buffering is scripted logs
> > 
> > > Ok, so is that a final 'no' for this whole feature, or is there is anything else I can do to get this feature in?
> > 
> > It is certainly not a "final no".  As I wrote in the very first comment, a) it's just a quick note, nothing more, and b) the feature is questionable.  If a good implementation will be submitted, we can consider committing it.
> > 
> That's good, I thought you were just rejecting politely :)

Well, I don't think there is a big difference, actually.

> It would be really great if you could point me to specific parts you think look bad.
> For example, I'm guessing that you don't like the callbacks I added to open file cache,
> but I was thinking that it's better to do it this way than to duplicate large chunks of code
> and write an open file cache specific to log, please let me know if you think otherwise.
> Any feedback you can provide will be appreciated

In no particular order:

- there are various style issues;

- you've dropped "simulate successful logging" comments which 
  where present for a reason;

- introducing separate ngx_open_file_cache_ext_t structure looks 
  completely pointless;

- it looks there are too many callbacks, it should be possible to 
  omit at least some of them.

Sorry, but please don't expect any further review / comments 
though.

-- 
Maxim Dounin
http://nginx.org/

From eran.kornblau at kaltura.com  Mon Aug 14 21:24:35 2017
From: eran.kornblau at kaltura.com (Eran Kornblau)
Date: Mon, 14 Aug 2017 21:24:35 +0000
Subject: Add support for buffering is scripted logs
In-Reply-To: <7AEF0D22-0BAF-42E3-9DE8-248987CD4111@gmail.com>
References: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814154507.GV93611@mdounin.ru>
 <AM4PR0401MB1731067E9B1437EA6184E779F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814165444.GX93611@mdounin.ru>
 <AM4PR0401MB173109E2D01BC0B95F72909EF58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814173346.GY93611@mdounin.ru>
 <AM4PR0401MB17314B45CBA8DA7F3EA2DA13F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <7AEF0D22-0BAF-42E3-9DE8-248987CD4111@gmail.com>
Message-ID: <AM4PR0401MB1731512FFBD7FDCBDB9966C7F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>

> -----Original Message-----
> From: nginx-devel [mailto:nginx-devel-bounces at nginx.org] On Behalf Of Alexey Ivanov
> Sent: Monday, August 14, 2017 9:25 PM
> To: nginx-devel at nginx.org
> Subject: Re: Add support for buffering is scripted logs
> 
> using syslog for that particular usecase seem way more elegant, customizable, and simple. As a side bonus you won't block event loop on vfs operations (open/write/close).
>
Thanks Alexey. 

Regarding the last point about performance, I ran a quick test, compared -
1. Nginx writing to gzip access log -
access_log /var/log/nginx/access_log.gz main gzip flush=5m;
2. Nginx writing to local rsyslog over UDP, which in turn writes to local file -
access_log syslog:server=127.0.0.1 main;
3. Nginx writing to remote rsyslog on the same LAN over UDP -
access_log syslog:server=192.168.11.94 main;

Ran this command (execute 10 apache bench tests, take the median value):
(for i in `seq 1 10`; do ab -c 1000 -n 100000 127.0.0.1/alive.html 2>/dev/null | grep '^  50%' ; done) | awk '{print $2}' | xargs

The nginx location being tested simply does 'return 200 "hello"'.

Results are:
1. 4 6 5 6 3 5 3 5 7 5
2. 6 7 5 7 6 7 7 7 7 7
3. 5 6 6 6 6 6 6 6 4 6

The numbers fluctuated a bit from one run to another (did it a few times), but the overall trend was the same - 
syslog is slower than having nginx write with gzip. The difference is not dramatic, but it's visible.
This makes sense since with buffered write, nginx only has to perform a syscall once every X messages, 
most messages are written without any syscalls.
Regarding what you wrote about open/write/close, when using variables (even before this patch), 
it is possible to enable open file cache on log files, so you only have 'write'.

Eran


From savetherbtz at gmail.com  Mon Aug 14 22:42:07 2017
From: savetherbtz at gmail.com (Alexey Ivanov)
Date: Mon, 14 Aug 2017 15:42:07 -0700
Subject: Add support for buffering is scripted logs
In-Reply-To: <AM4PR0401MB1731512FFBD7FDCBDB9966C7F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
References: <AM4PR0401MB17315BB2A20AAE0AF944BCB4F5B50@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814154507.GV93611@mdounin.ru>
 <AM4PR0401MB1731067E9B1437EA6184E779F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814165444.GX93611@mdounin.ru>
 <AM4PR0401MB173109E2D01BC0B95F72909EF58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <20170814173346.GY93611@mdounin.ru>
 <AM4PR0401MB17314B45CBA8DA7F3EA2DA13F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
 <7AEF0D22-0BAF-42E3-9DE8-248987CD4111@gmail.com>
 <AM4PR0401MB1731512FFBD7FDCBDB9966C7F58C0@AM4PR0401MB1731.eurprd04.prod.outlook.com>
Message-ID: <D4E6CEE3-7FA1-4718-9B28-B4DC0C8CE573@gmail.com>

You are indeed right about the open_file_cache, but you still left with a sync write(2) that can block, and blocking of eventloop is a major source of latency variance (unless you use Flash-based drives of course).

> On Aug 14, 2017, at 2:24 PM, Eran Kornblau <eran.kornblau at kaltura.com> wrote:
> 
>> -----Original Message-----
>> From: nginx-devel [mailto:nginx-devel-bounces at nginx.org] On Behalf Of Alexey Ivanov
>> Sent: Monday, August 14, 2017 9:25 PM
>> To: nginx-devel at nginx.org
>> Subject: Re: Add support for buffering is scripted logs
>> 
>> using syslog for that particular usecase seem way more elegant, customizable, and simple. As a side bonus you won't block event loop on vfs operations (open/write/close).
>> 
> Thanks Alexey.
> 
> Regarding the last point about performance, I ran a quick test, compared -
> 1. Nginx writing to gzip access log -
> access_log /var/log/nginx/access_log.gz main gzip flush=5m;
> 2. Nginx writing to local rsyslog over UDP, which in turn writes to local file -
> access_log syslog:server=127.0.0.1 main;
> 3. Nginx writing to remote rsyslog on the same LAN over UDP -
> access_log syslog:server=192.168.11.94 main;
> 
> Ran this command (execute 10 apache bench tests, take the median value):
> (for i in `seq 1 10`; do ab -c 1000 -n 100000 127.0.0.1/alive.html 2>/dev/null | grep '^  50%' ; done) | awk '{print $2}' | xargs
> 
> The nginx location being tested simply does 'return 200 "hello"'.
> 
> Results are:
> 1. 4 6 5 6 3 5 3 5 7 5
> 2. 6 7 5 7 6 7 7 7 7 7
> 3. 5 6 6 6 6 6 6 6 4 6
> 
> The numbers fluctuated a bit from one run to another (did it a few times), but the overall trend was the same -
> syslog is slower than having nginx write with gzip. The difference is not dramatic, but it's visible.
> This makes sense since with buffered write, nginx only has to perform a syscall once every X messages,
> most messages are written without any syscalls.
> Regarding what you wrote about open/write/close, when using variables (even before this patch),
> it is possible to enable open file cache on log files, so you only have 'write'.
> 
> Eran
> 
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 842 bytes
Desc: Message signed with OpenPGP
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170814/42033faa/attachment.bin>

From sdollberg at cloudflare.com  Tue Aug 15 10:10:10 2017
From: sdollberg at cloudflare.com (Stephan Dollberg)
Date: Tue, 15 Aug 2017 11:10:10 +0100
Subject: Heap buffer overflow (read) when using $binary_remote_addr with unix
 sockets
Message-ID: <CAC4b4PQ4bs7BPu0P5E+-H6EToT0vsJJaZa7jrFH8uWec9O=VVQ@mail.gmail.com>

Hi,

When using $binary_remote_addr together with unix sockets (without
using X-Real-Ip) there is a heap buffer overread of two bytes.

The problem is that we only allocate two bytes for c->sockaddr here
http://hg.nginx.org/nginx/file/tip/src/event/ngx_event_accept.c#l167
but later on assume it to be of size four
http://hg.nginx.org/nginx/file/tip/src/http/ngx_http_variables.c#l1246

Now, one could argue that using remote addr and unix sockets without
X-Real-Ip might not make sense but I still wanted to report it. Maybe
it might make sense to issue a warning or something.

The issue can be reproduced by compiling with address sanitizer and
-DNGX_DEBUG_PALLOC and using something like the following config:

daemon off;
master_process off;

events {
    worker_connections  1024;
}

error_log /dev/stdout debug;

http {
    server {
        listen 8000;
        listen unix:/tmp/nginx.sock backlog=10000;
        server_name  localhost;

        location / {
            proxy_pass http://unix:/tmp/nginx.sock:/foo;
        }

        location /foo {
            return 200 hello$binary_remote_addr;
        }
    }
}

Cheers,
Stephan

From xeioex at nginx.com  Tue Aug 15 18:20:22 2017
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 15 Aug 2017 18:20:22 +0000
Subject: [njs] Fixed processing files from stdin.
Message-ID: <hg.c0e8bd4964be.1502821222.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/c0e8bd4964be
branches:  
changeset: 397:c0e8bd4964be
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Aug 15 21:19:13 2017 +0300
description:
Fixed processing files from stdin.

diffstat:

 njs/njs.c |  4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diffs (14 lines):

diff -r f47b1eed6ed8 -r c0e8bd4964be njs/njs.c
--- a/njs/njs.c	Tue Aug 08 18:06:53 2017 +0300
+++ b/njs/njs.c	Tue Aug 15 21:19:13 2017 +0300
@@ -263,8 +263,8 @@ njs_process_file(njs_opts_t *opts, njs_v
                 return NXT_ERROR;
             }
 
-            p = script.start;
-            end = p + size;
+            p = script.start + script.length;
+            end = script.start + size;
         }
 
         memcpy(p, buf, n);

From xeioex at nginx.com  Mon Aug 21 10:45:42 2017
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Mon, 21 Aug 2017 10:45:42 +0000
Subject: [njs] Improved editline detection on various platforms.
Message-ID: <hg.4b11d945ce7d.1503312342.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/4b11d945ce7d
branches:  
changeset: 398:4b11d945ce7d
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Fri Aug 18 18:40:39 2017 +0300
description:
Improved editline detection on various platforms.

diffstat:

 Makefile          |   2 +-
 njs/njs.c         |   2 +-
 nxt/auto/editline |  23 +++++++++++++++++++----
 3 files changed, 21 insertions(+), 6 deletions(-)

diffs (69 lines):

diff -r c0e8bd4964be -r 4b11d945ce7d Makefile
--- a/Makefile	Tue Aug 15 21:19:13 2017 +0300
+++ b/Makefile	Fri Aug 18 18:40:39 2017 +0300
@@ -396,7 +396,7 @@ dist:
 	njs/njs.c \
 
 	$(NXT_CC) -o $(NXT_BUILDDIR)/njs $(NXT_CFLAGS) \
-		-I$(NXT_LIB) -Injs \
+		-I$(NXT_LIB) $(NXT_EDITLINE_CFLAGS) -Injs \
 		njs/njs.c \
 		$(NXT_BUILDDIR)/libnjs.a \
 		-lm $(NXT_PCRE_LIB) $(NXT_EDITLINE_LIB)
diff -r c0e8bd4964be -r 4b11d945ce7d njs/njs.c
--- a/njs/njs.c	Tue Aug 15 21:19:13 2017 +0300
+++ b/njs/njs.c	Fri Aug 18 18:40:39 2017 +0300
@@ -27,7 +27,7 @@
 #include <njs_variable.h>
 #include <njs_parser.h>
 
-#include <editline/readline.h>
+#include <readline.h>
 
 
 typedef enum {
diff -r c0e8bd4964be -r 4b11d945ce7d nxt/auto/editline
--- a/nxt/auto/editline	Tue Aug 15 21:19:13 2017 +0300
+++ b/nxt/auto/editline	Fri Aug 18 18:40:39 2017 +0300
@@ -1,12 +1,15 @@
 # Copyright (C) Dmitry Volyntsev
 # Copyright (C) NGINX, Inc.
 
+nxt_found=no
+
 nxt_feature="editline library"
 nxt_feature_name=NXT_HAVE_EDITLINE
 nxt_feature_run=no
-nxt_feature_incs=
-nxt_feature_libs=-ledit
-nxt_feature_test="#include <editline/readline.h>
+nxt_feature_incs="-I/usr/include/editline"
+nxt_feature_libs="-ledit"
+nxt_feature_test="#include <stdio.h>
+                  #include <readline.h>
 
                  int main(void) {
                      add_history(NULL);
@@ -14,9 +17,21 @@ nxt_feature_test="#include <editline/rea
                  }"
 . ${NXT_AUTO}feature
 
+if [ $nxt_found = no ]; then
+
+    # FreeBSD port
+
+    nxt_feature="editline in /usr/include/edit"
+    nxt_feature_incs="-I/usr/include/edit/readline"
+
+    . auto/feature
+fi
+
 if [ $nxt_found = yes ]; then
     cat << END >> $NXT_MAKEFILE_CONF
-NXT_EDITLINE_LIB = -ledit
+
+NXT_EDITLINE_CFLAGS = $nxt_feature_incs
+NXT_EDITLINE_LIB = $nxt_feature_libs
 END
 
 else

From mdounin at mdounin.ru  Mon Aug 21 23:40:26 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 22 Aug 2017 02:40:26 +0300
Subject: [PATCH 1 of 2] Core: add function to decode hexadecimal strings
In-Reply-To: <7e10e1dc1fae0f538a0a.1501267536@OLA-6J5NNS1.ad.garmin.com>
References: <patchbomb.1501267535@OLA-6J5NNS1.ad.garmin.com>
 <7e10e1dc1fae0f538a0a.1501267536@OLA-6J5NNS1.ad.garmin.com>
Message-ID: <20170821234026.GY93611@mdounin.ru>

Hello!

On Fri, Jul 28, 2017 at 01:45:36PM -0500, Nate Karstens wrote:

> # HG changeset patch
> # User Nate Karstens <nate.karstens at garmin.com>
> # Date 1501265787 18000
> #      Fri Jul 28 13:16:27 2017 -0500
> # Node ID 7e10e1dc1fae0f538a0a74fd6783f9b33a905933
> # Parent  72d3aefc2993a639cc8cddc94aefc7c4390ee252
> Core: add function to decode hexadecimal strings.
> 
> Adds functionality to convert a hexadecimal string into binary data.
> This will be used to decode PSKs stored in hexadecimal representation.
> 
> Signed-off-by: Nate Karstens <nate.karstens at garmin.com>
> 
> diff -r 72d3aefc2993 -r 7e10e1dc1fae src/core/ngx_string.c
> --- a/src/core/ngx_string.c     Wed Jul 26 13:13:51 2017 +0300
> +++ b/src/core/ngx_string.c     Fri Jul 28 13:16:27 2017 -0500
> @@ -1104,6 +1104,54 @@ ngx_hextoi(u_char *line, size_t n)
>  }
> 
> 
> +ngx_int_t
> +ngx_hex_decode(u_char *dst, size_t dlen, u_char *src, size_t slen)
> +{
> +    u_char     c, ch;
> +    ngx_int_t  len;
> +
> +    if ((slen & 1) || (dlen < (slen / 2))) {
> +        return NGX_ERROR;

It doesn't look like dlen is useful in the interface, at least in 
the for it is implemented.  It could make sense if the function 
would written in a way which allows decoding only some bytes of 
the source string, but a simple "buffer must be at least as large 
as half of the source string" prerequisite certainly doesn't need 
an additional parameter.  For example, ngx_hex_dump() doesn't have 
it.

> +    }
> +
> +    len = slen / 2;

Returning a length known in advice might not be a good idea 
either.  It is already known to the caller.  Rather, I would 
recommend returning NGX_OK as, for example, ngx_decode_base64() 
does.

> +
> +    while (slen > 0) {
> +        ch = *src;
> +        c = (u_char) (ch | 0x20);
> +
> +        if (ch >= '0' && ch <= '9') {
> +            *dst = ch - '0';
> +        } else if (c >= 'a' && c <= 'f') {
> +            *dst = c - 'a' + 10;

It might be a good idea to introduce a u_char variable on stack, 
similar to how it is done in ngx_http_parse_complex_uri(). 

It also might be a good idea to restructure the code to avoid 
meaningless lowercase operations for 0..9 characters.  And I see 
no reason to use two variables, "ch" and "c".

It might also need some additional empty lines to better match 
style, though the code is anyway needs to be rewritten.

> +        } else {
> +            return NGX_ERROR;
> +        }
> +
> +        *dst <<= 4;
> +        src++;

It might be more logical to combine src++ with reading the 
character,

   ch = *src++;

Such approach can be seen in many other functions in ngx_string.c.

> +
> +        ch = *src;
> +        c = (u_char) (ch | 0x20);
> +
> +        if (ch >= '0' && ch <= '9') {
> +            *dst |= ch - '0';
> +        } else if (c >= 'a' && c <= 'f') {
> +            *dst |= c - 'a' + 10;
> +        } else {
> +            return NGX_ERROR;
> +        }
> +
> +        dst++;
> +        src++;
> +
> +        slen -= 2;
> +    }
> +
> +    return len;
> +}
> +
> +
>  u_char *
>  ngx_hex_dump(u_char *dst, u_char *src, size_t len)
>  {
> diff -r 72d3aefc2993 -r 7e10e1dc1fae src/core/ngx_string.h
> --- a/src/core/ngx_string.h     Wed Jul 26 13:13:51 2017 +0300
> +++ b/src/core/ngx_string.h     Fri Jul 28 13:16:27 2017 -0500
> @@ -176,6 +176,7 @@ off_t ngx_atoof(u_char *line, size_t n);
>  time_t ngx_atotm(u_char *line, size_t n);
>  ngx_int_t ngx_hextoi(u_char *line, size_t n);
> 
> +ngx_int_t ngx_hex_decode(u_char *dst, size_t dlen, u_char *src, size_t slen);
>  u_char *ngx_hex_dump(u_char *dst, u_char *src, size_t len);

It might be a good idea to change the order, so the decode 
function is placed after an encode one, as in other similar 
functions in the same files.

-- 
Maxim Dounin
http://nginx.org/

From mdounin at mdounin.ru  Mon Aug 21 23:40:54 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 22 Aug 2017 02:40:54 +0300
Subject: [PATCH 2 of 2] SSL: add connection support
In-Reply-To: <9537b7d299131e41a3f5.1501267537@OLA-6J5NNS1.ad.garmin.com>
References: <patchbomb.1501267535@OLA-6J5NNS1.ad.garmin.com>
 <9537b7d299131e41a3f5.1501267537@OLA-6J5NNS1.ad.garmin.com>
Message-ID: <20170821234053.GZ93611@mdounin.ru>

Hello!

On Fri, Jul 28, 2017 at 01:45:37PM -0500, Nate Karstens wrote:

> # HG changeset patch
> # User Nate Karstens <nate.karstens at garmin.com>
> # Date 1501265849 18000
> #      Fri Jul 28 13:17:29 2017 -0500
> # Node ID 9537b7d299131e41a3f5993257000d328e28b117
> # Parent  7e10e1dc1fae0f538a0a74fd6783f9b33a905933
> SSL: add connection support.

It is not clear what this summary line is expected to mean.

> 
> Adds support for TLS connections using PSK cipher suites. A new
> configuration directive, ssl_psk_file, specifies the file that
> contains a list of identities and associated PSKs. Each line of
> the file begins with the identity, followed by a colon character
> (':'), and ending with the PSK. As required by RFC 4279 section
> 5.4, PSKs may be entered either as plain text or using hexadecimal
> encoding. Hexadecimal PSKs must begin with "{HEX}". PSKs without
> this prefix are assumed to be plain text, but they may optionally
> begin with "{PLAIN}" to denote this. Some examples:
> 
> gary:plain_text_password
> min:{PLAIN}another_text_password
> cliff:{HEX}ab0123CD

Looks good now, thanks.

> 
> PSK functionality can be easily tested with the OpenSSL s_client
> using the "-psk" and "-psk_identity" options.

Note that this stops working with OpenSSL 1.1.1 when using TLS 
1.3.  As of now, using PSK with TLS 1.3 in OpenSSL requires using 
a different callback (SSL_CTX_set_psk_find_session_callback()) 
with slightly different logic, and keys must match sha256 or 
sha384 length exactly.  An example code can be found in 
OpenSSL's apps/s_server.c.  This is probably something ok to 
postpone for later though.

> 
> Signed-off-by: Nate Karstens <nate.karstens at garmin.com>
> 
> diff -r 7e10e1dc1fae -r 9537b7d29913 contrib/vim/syntax/nginx.vim
> --- a/contrib/vim/syntax/nginx.vim      Fri Jul 28 13:16:27 2017 -0500
> +++ b/contrib/vim/syntax/nginx.vim      Fri Jul 28 13:17:29 2017 -0500
> @@ -550,6 +550,7 @@ syn keyword ngxDirective contained ssl_p
>  syn keyword ngxDirective contained ssl_prefer_server_ciphers
>  syn keyword ngxDirective contained ssl_preread
>  syn keyword ngxDirective contained ssl_protocols
> +syn keyword ngxDirective contained ssl_psk_file
>  syn keyword ngxDirective contained ssl_session_cache
>  syn keyword ngxDirective contained ssl_session_ticket_key
>  syn keyword ngxDirective contained ssl_session_tickets
> diff -r 7e10e1dc1fae -r 9537b7d29913 src/event/ngx_event_openssl.c
> --- a/src/event/ngx_event_openssl.c     Fri Jul 28 13:16:27 2017 -0500
> +++ b/src/event/ngx_event_openssl.c     Fri Jul 28 13:17:29 2017 -0500
> @@ -11,6 +11,7 @@
> 
> 
>  #define NGX_SSL_PASSWORD_BUFFER_SIZE  4096
> +#define NGX_SSL_PSK_BUFFER_SIZE       4096
> 
> 
>  typedef struct {
> @@ -23,6 +24,8 @@ static int ngx_ssl_password_callback(cha
>  static int ngx_ssl_verify_callback(int ok, X509_STORE_CTX *x509_store);
>  static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where,
>      int ret);
> +static unsigned int ngx_ssl_psk_callback(ngx_ssl_conn_t *ssl_conn,
> +    const char *identity, unsigned char *psk, unsigned int max_psk_len);

This callback probably should be placed after the 
ngx_ssl_psk_file() function which uses it.  And see below about 
ngx_ssl_psk_file() position.

>  static void ngx_ssl_passwords_cleanup(void *data);
>  static void ngx_ssl_handshake_handler(ngx_event_t *ev);
>  static ngx_int_t ngx_ssl_handle_recv(ngx_connection_t *c, int n);
> @@ -114,6 +117,7 @@ int  ngx_ssl_certificate_index;
>  int  ngx_ssl_next_certificate_index;
>  int  ngx_ssl_certificate_name_index;
>  int  ngx_ssl_stapling_index;
> +int  ngx_ssl_psk_index;
> 
> 
>  ngx_int_t
> @@ -225,6 +229,13 @@ ngx_ssl_init(ngx_log_t *log)
>          return NGX_ERROR;
>      }
> 
> +    ngx_ssl_psk_index = SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL);

You are allocating connection-specific index (for SSL structures), but use it
as context-specific one (in SSL_CTX structures).  This is not going to work.

Also, please note that indexes are orderd based on their type.  
This particular position is wrong for both the type you've used 
and the type it should be.

> +
> +    if (ngx_ssl_psk_index == -1) {
> +        ngx_ssl_error(NGX_LOG_ALERT, log, 0, "SSL_get_ex_new_index() failed");
> +        return NGX_ERROR;
> +    }
> +
>      return NGX_OK;
>  }
> 
> @@ -875,6 +886,138 @@ ngx_ssl_info_callback(const ngx_ssl_conn
>  }
> 
> 
> +static unsigned int
> +ngx_ssl_psk_callback(ngx_ssl_conn_t *ssl_conn, const char *identity,
> +    unsigned char *psk, unsigned int max_psk_len)
> +{
> +    u_char             *p, *last, *end, *psk_ptr;
> +    size_t              len;
> +    ssize_t             n;
> +    SSL_CTX            *ssl_ctx;
> +    ngx_fd_t            fd;
> +    ngx_str_t          *psk_file;
> +    ngx_int_t           psk_len;
> +    ngx_connection_t   *c;
> +    u_char              buf[NGX_SSL_PSK_BUFFER_SIZE];
> +
> +    c = ngx_ssl_get_connection(ssl_conn);
> +
> +    ssl_ctx = c->ssl->session_ctx;

Any reasons to use session-specific context here?  This will 
preclude use of virtualhost-specific PSK files, which looks like a 
bad idea.  Not to mention that the mere existence of 
session-specific contexts in c->ssl is a workaround for the 
OpenSSL session resumption logic, see 
http://hg.nginx.org/nginx/rev/97f102a13f33.

Unless there are reasons, normal SSL context should be used 
instead, as available via SSL_get_SSL_CTX().  Note though that for 
proper virtualhost support additional changes in 
ngx_http_ssl_servername() might be also needed, especially if 
callbacks will be set conditionally (see below), as OpenSSL might 
not adjust callbacks properly on SSL_set_SSL_CTX().

> +    psk_file = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_psk_index);

The "psk_file" name looks excessive.  Just "file" would be fine 
here.

> +
> +    if (psk_file == NULL) {
> +        return 0;
> +    }

As the code only configures callback with a PSK file, 
this should never happen.

> +
> +    fd = ngx_open_file(psk_file->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
> +    if (fd == NGX_INVALID_FILE) {
> +        ngx_ssl_error(NGX_LOG_ERR, c->log, ngx_errno,
> +                      ngx_open_file_n " \"%V\" failed", psk_file);
> +        return 0;
> +    }
> +
> +    len = 0;
> +    last = buf;
> +
> +    do {
> +        n = ngx_read_fd(fd, last, NGX_SSL_PSK_BUFFER_SIZE - len);
> +
> +        if (n == -1) {
> +            ngx_ssl_error(NGX_LOG_ERR, c->log, ngx_errno,
> +                          ngx_read_fd_n " \"%V\" failed", psk_file);
> +            psk_len = 0;
> +            goto cleanup;
> +        }
> +
> +        end = last + n;
> +
> +        if (len && n == 0) {
> +            *end++ = LF;
> +        }
> +
> +        for (p = buf; /* void */; p = last) {
> +            last = ngx_strlchr(last, end, LF);
> +
> +            if (last == NULL) {
> +                break;
> +            }
> +
> +            len = last++ - p;
> +
> +            if (len && p[len - 1] == CR) {
> +                len--;
> +            }
> +
> +            if (len == 0) {
> +                continue;
> +            }
> +
> +            psk_ptr = (u_char *) ngx_strlchr(p, p + len, ':');

There are no reasons for the cast here, ngx_strlchr() returns 
u_char *.

The "psk_ptr" name looks excessive.  Instead, I would recommend 
using something like "colon" and moving the "p" pointer to 
colon + 1 after identity check.

> +
> +            if (psk_ptr == NULL) {
> +                continue;
> +            }
> +
> +            *psk_ptr = '\0';
> +            psk_ptr++;
> +
> +            if (ngx_strcmp(p, identity) != 0) {
> +                continue;
> +            }
> +
> +            if (ngx_strncmp(psk_ptr, "{HEX}", ngx_strlen("{HEX}")) == 0) {
> +                psk_ptr += ngx_strlen("{HEX}");

Using ngx_strlen() with static strings is a bad idea.  Instead, 
sizeof(...) - 1 should be used, much like in other parts of the 
code.

> +                psk_len = ngx_hex_decode(psk, max_psk_len, psk_ptr,
> +                                         p + len - psk_ptr);
> +                if (psk_len == NGX_ERROR) {
> +                    ngx_memzero(psk, max_psk_len);
> +                    psk_len = 0;
> +                }
> +                goto cleanup;

Style: there should be more empty lines here, including one before 
"} else".  I would recommend something like this:

            if (ngx_strncmp(psk_ptr, "{HEX}", sizeof("{HEX}") - 1) == 0) {
                psk_ptr += sizeof("{HEX}") - 1;

                psk_len = ngx_hex_decode(psk, max_psk_len, psk_ptr,
                                         p + len - psk_ptr);
                if (psk_len == NGX_ERROR) {
                    ngx_memzero(psk, max_psk_len);
                    psk_len = 0;
                }

                goto cleanup;

            } else ...

> +            } else if (ngx_strncmp(psk_ptr, "{PLAIN}",
> +                       ngx_strlen("{PLAIN}")) == 0) {

Style: indentation is wrong, ngx_strlen() should be aligned with 
ngx_strncmp() arguments, "== 0" should be on its own line and 
indented to the function checked, and "{" should be on its own 
line.

And since there is no need to wrap ngx_strcmp() arguments, the 
result should look like:
        
            } else if (ngx_strncmp(psk_ptr, "{PLAIN}", sizeof("{PLAIN}") - 1)
                       == 0)
            {


> +                psk_ptr += ngx_strlen("{PLAIN}");
> +            }
> +
> +            psk_len = last - 1 - psk_ptr;

This logic is hardly readable and wrong.  Notably, it will include 
trailing "CR" into the psk_len, despite the fact that the above 
code tries to ignore it.

Instead, I would recommend to adjast "p" as suggested above, and 
adjust "len" at the same time.  This will allow to get rid of such 
error-prone calculations, and simply use "p" and "len" here and in 
other places.

> +
> +            if (psk_len > max_psk_len) {

Compilation with at least clang fails here with the following 
message:

src/event/ngx_event_openssl.c:984:25: error: comparison of integers of different
      signs: 'ngx_int_t' (aka 'int') and 'unsigned int' [-Werror,-Wsign-compare]
            if (psk_len > max_psk_len) {
                ~~~~~~~ ^ ~~~~~~~~~~~

Changing the psk_len type to "unsigned int" as it should be 
resolves this (though introduces a different problem elsewhere, to 
be resolved with ngx_hex_decode() API improvements as discussed in 
the previous patch review).

> +                psk_len = 0;
> +                goto cleanup;
> +            }
> +
> +            ngx_memcpy(psk, psk_ptr, psk_len);
> +            goto cleanup;
> +        }
> +
> +        len = end - p;
> +
> +        if (len == NGX_SSL_PSK_BUFFER_SIZE) {
> +            ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
> +                          "too long line in \"%V\"", psk_file);
> +            psk_len = 0;
> +            goto cleanup;
> +        }
> +
> +        ngx_memmove(buf, p, len);
> +        last = buf + len;
> +
> +    } while (n != 0);
> +
> +cleanup:
> +
> +    if (ngx_close_file(fd) == NGX_FILE_ERROR) {
> +        ngx_ssl_error(NGX_LOG_ALERT, c->log, ngx_errno,
> +                      ngx_close_file_n " %V failed", psk_file);
> +        psk_len = 0;
> +    }
> +
> +    ngx_memzero(buf, NGX_SSL_PSK_BUFFER_SIZE);
> +
> +    return psk_len;

Compilation fails here with the following message:

src/event/ngx_event_openssl.c:1017:12: error: variable 'psk_len' may be
      uninitialized when used here [-Werror,-Wconditional-uninitialized]
    return psk_len;
           ^~~~~~~

This is indeed a bug in the code, as psk_len is never initialized 
if the identity is not found in the PSK file.

> +}
> +
> +
>  RSA *
>  ngx_ssl_rsa512_key_callback(ngx_ssl_conn_t *ssl_conn, int is_export,
>      int key_length)
> @@ -3137,6 +3280,23 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *
>  #endif
> 
> 
> +ngx_int_t
> +ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file)
> +{
> +#if OPENSSL_VERSION_NUMBER >= 0x1000000fL

This is not enough to compile with older OpenSSL versions, as 
there will be an unused ngx_ssl_psk_callback() function:

src/event/ngx_event_openssl.c:890:1: error: unused function
      'ngx_ssl_psk_callback' [-Werror,-Wunused-function]
ngx_ssl_psk_callback(ngx_ssl_conn_t *ssl_conn, const char *identity,
^

This also breaks builds with at least LibreSSL 2.5.5, as it does 
not have SSL_CTX_set_ex_data() defined, yet provides large enough 
OPENSSL_VERSION_NUMBER.  Checking for some related define 
(PSK_MAX_IDENTITY_LEN?) might be a better option.

Also, there should be empty lines after #if and before #endif, as 
the code inside #if/#endif is large enough and contains empty 
lines.

> +    if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_psk_index, file) == 0) {

This uses "file" as written in the configuration.  Instead, it 
should resolve the name using the ngx_conf_full_name() function, 
to avoid dependencies on the working directory.  See 
ngx_ssl_dhparam() for an example.

> +        ngx_ssl_error(NGX_LOG_ALERT, ssl->log, 0,
> +                      "SSL_CTX_set_ex_data() failed");

The NGX_LOG_ALERT logging level is not appropriate here.  As the 
error is fatal and will prevent nginx from starting, it should be
NGX_LOG_EMERG.

> +        return NGX_ERROR;
> +    }
> +
> +    SSL_CTX_set_psk_server_callback(ssl->ctx, ngx_ssl_psk_callback);

Always configuring a callback is probably not a good idea.  
Instead, there should be a check if file is not empty.

In particular, this will result in cryptic messages like:

... [error] ... open() "" failed (SSL:) (2: No such file or directory) while SSL handshaking

on servers without PSK file configured if a client tries to 
use PSK.

It also results in a critictal error like:

... [crit] ... SSL_do_handshake() failed (SSL: error:1419E0DF:SSL routines:tls_process_cke_psk_preamble:psk identity not found) while SSL handshaking

which suggests that the patch needs to adjust severity levels in 
ngx_ssl_connection_error().

> +#endif
> +
> +    return NGX_OK;
> +}
> +
> +
>  void
>  ngx_ssl_cleanup_ctx(void *data)
>  {
> diff -r 7e10e1dc1fae -r 9537b7d29913 src/event/ngx_event_openssl.h
> --- a/src/event/ngx_event_openssl.h     Fri Jul 28 13:16:27 2017 -0500
> +++ b/src/event/ngx_event_openssl.h     Fri Jul 28 13:17:29 2017 -0500
> @@ -171,6 +171,7 @@ ngx_int_t ngx_ssl_session_cache(ngx_ssl_
>      ssize_t builtin_session_cache, ngx_shm_zone_t *shm_zone, time_t timeout);
>  ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl,
>      ngx_array_t *paths);
> +ngx_int_t ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
>  ngx_int_t ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data);
>  ngx_int_t ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c,
>      ngx_uint_t flags);

The position chosen for for the function looks suboptimal, as it 
is placed between functions related to session caching.  

> @@ -255,6 +256,7 @@ extern int  ngx_ssl_certificate_index;
>  extern int  ngx_ssl_next_certificate_index;
>  extern int  ngx_ssl_certificate_name_index;
>  extern int  ngx_ssl_stapling_index;
> +extern int  ngx_ssl_psk_index;
> 
> 
>  #endif /* _NGX_EVENT_OPENSSL_H_INCLUDED_ */
> diff -r 7e10e1dc1fae -r 9537b7d29913 src/http/modules/ngx_http_ssl_module.c
> --- a/src/http/modules/ngx_http_ssl_module.c    Fri Jul 28 13:16:27 2017 -0500
> +++ b/src/http/modules/ngx_http_ssl_module.c    Fri Jul 28 13:17:29 2017 -0500
> @@ -234,6 +234,13 @@ static ngx_command_t  ngx_http_ssl_comma
>        offsetof(ngx_http_ssl_srv_conf_t, stapling_verify),
>        NULL },
> 
> +    { ngx_string("ssl_psk_file"),
> +      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
> +      ngx_conf_set_str_slot,
> +      NGX_HTTP_SRV_CONF_OFFSET,
> +      offsetof(ngx_http_ssl_srv_conf_t, psk_file),
> +      NULL },
> +
>        ngx_null_command
>  };
> 
> @@ -539,6 +546,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t
>       *     sscf->shm_zone = NULL;
>       *     sscf->stapling_file = { 0, NULL };
>       *     sscf->stapling_responder = { 0, NULL };
> +     *     sscf->psk_file = { 0, NULL };
>       */
> 
>      sscf->enable = NGX_CONF_UNSET;
> @@ -620,6 +628,8 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *
>      ngx_conf_merge_str_value(conf->stapling_responder,
>                           prev->stapling_responder, "");
> 
> +    ngx_conf_merge_str_value(conf->psk_file, prev->psk_file, "");
> +
>      conf->ssl.log = cf->log;
> 
>      if (conf->enable) {
> @@ -800,6 +810,12 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *
> 
>      }
> 
> +    if (ngx_ssl_psk_file(cf, &conf->ssl, &conf->psk_file)
> +        != NGX_OK)
> +    {
> +        return NGX_CONF_ERROR;
> +    }
> +

Style: there is no need to wrap lines here, just using

    if (ngx_ssl_psk_file(cf, &conf->ssl, &conf->psk_file) != NGX_OK) {
        return NGX_CONF_ERROR;
    }

would be fine.

>      return NGX_CONF_OK;
>  }
> 
> diff -r 7e10e1dc1fae -r 9537b7d29913 src/http/modules/ngx_http_ssl_module.h
> --- a/src/http/modules/ngx_http_ssl_module.h    Fri Jul 28 13:16:27 2017 -0500
> +++ b/src/http/modules/ngx_http_ssl_module.h    Fri Jul 28 13:17:29 2017 -0500
> @@ -55,6 +55,8 @@ typedef struct {
>      ngx_str_t                       stapling_file;
>      ngx_str_t                       stapling_responder;
> 
> +    ngx_str_t                       psk_file;
> +
>      u_char                         *file;
>      ngx_uint_t                      line;
>  } ngx_http_ssl_srv_conf_t;
> 
> ________________________________
> 
> CONFIDENTIALITY NOTICE: This email and any attachments are for the sole use of the intended recipient(s) and contain information that may be Garmin confidential and/or Garmin legally privileged. If you have received this email in error, please notify the sender by reply email and delete the message. Any disclosure, copying, distribution or use of this communication (including attachments) by someone other than the intended recipient is prohibited. Thank you.
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

-- 
Maxim Dounin
http://nginx.org/

From mdounin at mdounin.ru  Mon Aug 21 23:41:59 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 22 Aug 2017 02:41:59 +0300
Subject: [PATCH] SSL: add PSK identity variable
In-Reply-To: <00a1466fe33b8969ef76.1501267753@OLA-6J5NNS1.ad.garmin.com>
References: <010248>
 <00a1466fe33b8969ef76.1501267753@OLA-6J5NNS1.ad.garmin.com>
Message-ID: <20170821234159.GA93611@mdounin.ru>

Hello!

On Fri, Jul 28, 2017 at 01:49:13PM -0500, Nate Karstens wrote:

> # HG changeset patch
> # User Nate Karstens <nate.karstens at garmin.com>
> # Date 1501265895 18000
> #      Fri Jul 28 13:18:15 2017 -0500
> # Node ID 00a1466fe33b8969ef765d8d0547dfbc7c97dd4e
> # Parent  9537b7d299131e41a3f5993257000d328e28b117
> SSL: add PSK identity variable.
> 
> Adds the variable $ssl_psk_identity to get the PSK identity
> used in a connnection secured with a PSK cipher suite.
> 
> Signed-off-by: Nate Karstens <nate.karstens at garmin.com>
> 
> diff -r 9537b7d29913 -r 00a1466fe33b src/event/ngx_event_openssl.c
> --- a/src/event/ngx_event_openssl.c     Fri Jul 28 13:17:29 2017 -0500
> +++ b/src/event/ngx_event_openssl.c     Fri Jul 28 13:18:15 2017 -0500
> @@ -4249,6 +4249,22 @@ ngx_ssl_get_client_v_remain(ngx_connecti
>  }
> 
> 
> +ngx_int_t
> +ngx_ssl_get_psk_identity(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s)
> +{

Following ngx_ssl_parse_time() is a helper function for 
ngx_ssl_get_client_v_remain(), so it might be a good idea to put 
this function after ngx_ssl_parse_time().  YMMV.

> +    const char  *identity;
> +
> +    identity = SSL_get_psk_identity(c->ssl->connection);

This needs conditional compilation.

> +
> +    if (identity) {
> +        s->data = (u_char *) identity;
> +        s->len = ngx_strlen(identity);

Directly returning the result of SSL_get_psk_identity() without 
copying it might not be safe.  The pointer returned is allocated in 
the context of the SSL session, and it can be freed at any time 
due to renegotiation (unfortunately, OpenSSL still allows 
renegotiation and provides no way to prevent it).  Instead, the 
value should be copied to memory allocated from the pool.

> +    }
> +
> +    return NGX_OK;
> +}
> +
> +
>  static time_t
>  ngx_ssl_parse_time(
>  #if OPENSSL_VERSION_NUMBER > 0x10100000L
> diff -r 9537b7d29913 -r 00a1466fe33b src/event/ngx_event_openssl.h
> --- a/src/event/ngx_event_openssl.h     Fri Jul 28 13:17:29 2017 -0500
> +++ b/src/event/ngx_event_openssl.h     Fri Jul 28 13:18:15 2017 -0500
> @@ -233,6 +233,8 @@ ngx_int_t ngx_ssl_get_client_v_end(ngx_c
>      ngx_str_t *s);
>  ngx_int_t ngx_ssl_get_client_v_remain(ngx_connection_t *c, ngx_pool_t *pool,
>      ngx_str_t *s);
> +ngx_int_t ngx_ssl_get_psk_identity(ngx_connection_t *c, ngx_pool_t *pool,
> +    ngx_str_t *s);
> 
> 
>  ngx_int_t ngx_ssl_handshake(ngx_connection_t *c);
> diff -r 9537b7d29913 -r 00a1466fe33b src/http/modules/ngx_http_ssl_module.c
> --- a/src/http/modules/ngx_http_ssl_module.c    Fri Jul 28 13:17:29 2017 -0500
> +++ b/src/http/modules/ngx_http_ssl_module.c    Fri Jul 28 13:18:15 2017 -0500
> @@ -336,6 +336,9 @@ static ngx_http_variable_t  ngx_http_ssl
>      { ngx_string("ssl_client_v_remain"), NULL, ngx_http_ssl_variable,
>        (uintptr_t) ngx_ssl_get_client_v_remain, NGX_HTTP_VAR_CHANGEABLE, 0 },
> 
> +    { ngx_string("ssl_psk_identity"), NULL, ngx_http_ssl_variable,
> +      (uintptr_t) ngx_ssl_get_psk_identity, NGX_HTTP_VAR_CHANGEABLE, 0 },
> +
>      { ngx_null_string, NULL, NULL, 0, 0, 0 }
>  };
> 
> 
> ________________________________
> 
> CONFIDENTIALITY NOTICE: This email and any attachments are for the sole use of the intended recipient(s) and contain information that may be Garmin confidential and/or Garmin legally privileged. If you have received this email in error, please notify the sender by reply email and delete the message. Any disclosure, copying, distribution or use of this communication (including attachments) by someone other than the intended recipient is prohibited. Thank you.
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

-- 
Maxim Dounin
http://nginx.org/

From mdounin at mdounin.ru  Mon Aug 21 23:42:39 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 22 Aug 2017 02:42:39 +0300
Subject: [PATCH] SSL: add identity hint config directive
In-Reply-To: <d47b57ebf82c1eedb423.1501267844@OLA-6J5NNS1.ad.garmin.com>
References: <010249>
 <d47b57ebf82c1eedb423.1501267844@OLA-6J5NNS1.ad.garmin.com>
Message-ID: <20170821234239.GB93611@mdounin.ru>

Hello!

On Fri, Jul 28, 2017 at 01:50:44PM -0500, Nate Karstens wrote:

> # HG changeset patch
> # User Nate Karstens <nate.karstens at garmin.com>
> # Date 1501265943 18000
> #      Fri Jul 28 13:19:03 2017 -0500
> # Node ID d47b57ebf82c1eedb4236a661b9d786dfd06b468
> # Parent  00a1466fe33b8969ef765d8d0547dfbc7c97dd4e
> SSL: add identity hint config directive.
> 
> Adds the directive "ssl_psk_identity_hint" to the ngx_http_ssl_module.
> This allows the user to specify the PSK identity hint given to the
> connecting client.
> 
> Signed-off-by: Nate Karstens <nate.karstens at garmin.com>
> 
> diff -r 00a1466fe33b -r d47b57ebf82c contrib/vim/syntax/nginx.vim
> --- a/contrib/vim/syntax/nginx.vim      Fri Jul 28 13:18:15 2017 -0500
> +++ b/contrib/vim/syntax/nginx.vim      Fri Jul 28 13:19:03 2017 -0500
> @@ -551,6 +551,7 @@ syn keyword ngxDirective contained ssl_p
>  syn keyword ngxDirective contained ssl_preread
>  syn keyword ngxDirective contained ssl_protocols
>  syn keyword ngxDirective contained ssl_psk_file
> +syn keyword ngxDirective contained ssl_psk_identity_hint
>  syn keyword ngxDirective contained ssl_session_cache
>  syn keyword ngxDirective contained ssl_session_ticket_key
>  syn keyword ngxDirective contained ssl_session_tickets
> diff -r 00a1466fe33b -r d47b57ebf82c src/event/ngx_event_openssl.c
> --- a/src/event/ngx_event_openssl.c     Fri Jul 28 13:18:15 2017 -0500
> +++ b/src/event/ngx_event_openssl.c     Fri Jul 28 13:19:03 2017 -0500
> @@ -3281,7 +3281,8 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *
> 
> 
>  ngx_int_t
> -ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file)
> +ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file,
> +    ngx_str_t *identity_hint)
>  {
>  #if OPENSSL_VERSION_NUMBER >= 0x1000000fL
>      if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_psk_index, file) == 0) {
> @@ -3290,6 +3291,13 @@ ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl
>          return NGX_ERROR;
>      }
> 
> +    if (SSL_CTX_use_psk_identity_hint(ssl->ctx,
> +                                      (char *) identity_hint->data) == 0) {

Style: "== 0)" and "{" should be on their own lines, no need to 
wrap SSL_CTX_use_psk_identity_hint() arguments as they fit into 80 
chars:

    if (SSL_CTX_use_psk_identity_hint(ssl->ctx, (char *) identity_hint->data)
        == 0)
    {

> +        ngx_ssl_error(NGX_LOG_ALERT, ssl->log, 0,
> +                      "SSL_CTX_use_psk_identity_hint() failed");

The NGX_LOG_ALERT logging level is not appropriate here.  As the 
error is fatal and will prevent nginx from starting, it should be 
NGX_LOG_EMERG.

> +        return NGX_ERROR;
> +    }
> +
>      SSL_CTX_set_psk_server_callback(ssl->ctx, ngx_ssl_psk_callback);
>  #endif
> 
> diff -r 00a1466fe33b -r d47b57ebf82c src/event/ngx_event_openssl.h
> --- a/src/event/ngx_event_openssl.h     Fri Jul 28 13:18:15 2017 -0500
> +++ b/src/event/ngx_event_openssl.h     Fri Jul 28 13:19:03 2017 -0500
> @@ -171,7 +171,8 @@ ngx_int_t ngx_ssl_session_cache(ngx_ssl_
>      ssize_t builtin_session_cache, ngx_shm_zone_t *shm_zone, time_t timeout);
>  ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl,
>      ngx_array_t *paths);
> -ngx_int_t ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
> +ngx_int_t ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file,
> +    ngx_str_t *identity_hint);
>  ngx_int_t ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data);
>  ngx_int_t ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c,
>      ngx_uint_t flags);
> diff -r 00a1466fe33b -r d47b57ebf82c src/http/modules/ngx_http_ssl_module.c
> --- a/src/http/modules/ngx_http_ssl_module.c    Fri Jul 28 13:18:15 2017 -0500
> +++ b/src/http/modules/ngx_http_ssl_module.c    Fri Jul 28 13:19:03 2017 -0500
> @@ -241,6 +241,13 @@ static ngx_command_t  ngx_http_ssl_comma
>        offsetof(ngx_http_ssl_srv_conf_t, psk_file),
>        NULL },
> 
> +    { ngx_string("ssl_psk_identity_hint"),
> +      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
> +      ngx_conf_set_str_slot,
> +      NGX_HTTP_SRV_CONF_OFFSET,
> +      offsetof(ngx_http_ssl_srv_conf_t, psk_identity_hint),
> +      NULL },
> +
>        ngx_null_command
>  };
> 
> @@ -550,6 +557,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t
>       *     sscf->stapling_file = { 0, NULL };
>       *     sscf->stapling_responder = { 0, NULL };
>       *     sscf->psk_file = { 0, NULL };
> +     *     sscf->psk_identity_hint = { 0, NULL };
>       */
> 
>      sscf->enable = NGX_CONF_UNSET;
> @@ -632,6 +640,7 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *
>                           prev->stapling_responder, "");
> 
>      ngx_conf_merge_str_value(conf->psk_file, prev->psk_file, "");
> +    ngx_conf_merge_str_value(conf->psk_identity_hint, prev->psk_identity_hint, "");

Style: lines should be under 80 chars.

> 
>      conf->ssl.log = cf->log;
> 
> @@ -813,7 +822,8 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *
> 
>      }
> 
> -    if (ngx_ssl_psk_file(cf, &conf->ssl, &conf->psk_file)
> +    if (ngx_ssl_psk_file(cf, &conf->ssl, &conf->psk_file,
> +                         &conf->psk_identity_hint)
>          != NGX_OK)
>      {
>          return NGX_CONF_ERROR;
> diff -r 00a1466fe33b -r d47b57ebf82c src/http/modules/ngx_http_ssl_module.h
> --- a/src/http/modules/ngx_http_ssl_module.h    Fri Jul 28 13:18:15 2017 -0500
> +++ b/src/http/modules/ngx_http_ssl_module.h    Fri Jul 28 13:19:03 2017 -0500
> @@ -56,6 +56,7 @@ typedef struct {
>      ngx_str_t                       stapling_responder;
> 
>      ngx_str_t                       psk_file;
> +    ngx_str_t                       psk_identity_hint;
> 
>      u_char                         *file;
>      ngx_uint_t                      line;
> 
> ________________________________
> 
> CONFIDENTIALITY NOTICE: This email and any attachments are for the sole use of the intended recipient(s) and contain information that may be Garmin confidential and/or Garmin legally privileged. If you have received this email in error, please notify the sender by reply email and delete the message. Any disclosure, copying, distribution or use of this communication (including attachments) by someone other than the intended recipient is prohibited. Thank you.
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

-- 
Maxim Dounin
http://nginx.org/

From thibaultcha at fastmail.com  Tue Aug 22 06:44:42 2017
From: thibaultcha at fastmail.com (Thibault Charbonnier)
Date: Mon, 21 Aug 2017 23:44:42 -0700
Subject: [PATCH] Resolver: parse hosts file entries
In-Reply-To: <24b3710c-c9c6-3995-c076-6adb00ae03a2@fastmail.com>
References: <db7373b7-cb57-a417-7f9f-47edbc074751@fastmail.com>
 <56e01788-baca-8c2d-c0ca-de1e02965226@fastmail.com>
 <24b3710c-c9c6-3995-c076-6adb00ae03a2@fastmail.com>
Message-ID: <74e7e4b4-3583-990e-2e44-429b16393c79@fastmail.com>

In case this patch interests anyone, here is an updated version of it.

It is fully backwards compatible; as long as the new 'hostsfile' option 
is not set, resolvers will behave as usual.

I believe it delivers on the promised features, but if it is lacking 
anything that prevents it from being merged, I would be glad to improve 
it (for now, the tests have been written with Test::Nginx only).

Best,
Thibault

# HG changeset patch
# User Thibault Charbonnier <thibaultcha at me.com>
# Date 1488252201 28800
#      Mon Feb 27 19:23:21 2017 -0800
# Node ID 558041ef1d70689ccc4c12f2487c3f75e6bbbccc
# Parent  a2f5e25d6a283546f76435b9fc3e7e814b092bae
Resolver: parse hosts file entries

The resolver directive can now take an optional 'hostsfile=<path>' option,
such as:

     resolver 8.8.4.4 hostsfile=/etc/hosts;

Hosts parsed from the hosts file are considered valid forever. The behavior
tries to be conservative, and only parses the hosts file when the option is
provided, to enforce backwards compatibility.

Additionally, this patch makes the resolver able to handle a host file 
in the
absence of a nameserver, like so:

     resolver hostsfile=/etc/hosts;

The 'hostsfile' option also honors the 'ipv6' flag of the 'resolver'
directive, as in:

     resolver 8.8.4.4 hostsfile=/etc/hosts ipv6=off;

diff -r a2f5e25d6a28 -r 558041ef1d70 src/core/ngx_resolver.c
--- a/src/core/ngx_resolver.c    Thu Aug 10 22:21:23 2017 +0300
+++ b/src/core/ngx_resolver.c    Mon Feb 27 19:23:21 2017 -0800
@@ -9,11 +9,12 @@
  #include <ngx_core.h>
  #include <ngx_event.h>

-
-#define NGX_RESOLVER_UDP_SIZE   4096
-
-#define NGX_RESOLVER_TCP_RSIZE  (2 + 65535)
-#define NGX_RESOLVER_TCP_WSIZE  8192
+#define NGX_RESOLVER_HOSTSFILE_BUFFER_SIZE  4096
+
+#define NGX_RESOLVER_UDP_SIZE               4096
+
+#define NGX_RESOLVER_TCP_RSIZE              (2 + 65535)
+#define NGX_RESOLVER_TCP_WSIZE              8192


  typedef struct {
@@ -122,6 +123,8 @@
      ngx_resolver_node_t *rn);
  static void ngx_resolver_srv_names_handler(ngx_resolver_ctx_t *ctx);
  static ngx_int_t ngx_resolver_cmp_srvs(const void *one, const void *two);
+static ngx_int_t ngx_resolver_parse_hosts_file(ngx_conf_t *cf,
+    ngx_resolver_t *r);

  #if (NGX_HAVE_INET6)
  static void ngx_resolver_rbtree_insert_addr6_value(ngx_rbtree_node_t 
*temp,
@@ -130,7 +133,6 @@
      struct in6_addr *addr, uint32_t hash);
  #endif

-
  ngx_resolver_t *
  ngx_resolver_create(ngx_conf_t *cf, ngx_str_t *names, ngx_uint_t n)
  {
@@ -246,6 +248,20 @@
          }
  #endif

+        if (ngx_strncmp(names[i].data, "hostsfile=", 10) == 0) {
+            r->hosts_file.len = names[i].len - 10;
+
+            if (r->hosts_file.len == 0) {
+                ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                                   "invalid parameter: %V", &names[i]);
+                return NULL;
+            }
+
+            r->hosts_file.data = names[i].data + 10;
+
+            continue;
+        }
+
          ngx_memzero(&u, sizeof(ngx_url_t));

          u.url = names[i];
@@ -276,6 +292,13 @@
          }
      }

+    if (r->hosts_file.len > 0
+        && ngx_resolver_parse_hosts_file(cf, r)
+        != NGX_OK)
+    {
+        return NULL;
+    }
+
      return r;
  }

@@ -396,7 +419,7 @@
          }
      }

-    if (r->connections.nelts == 0) {
+    if (r->connections.nelts == 0 && !r->hosts_file.len) {
          return NGX_NO_RESOLVER;
      }

@@ -807,6 +830,13 @@
  #endif

          ngx_rbtree_insert(tree, &rn->node);
+
+        if (r->connections.nelts == 0) {
+            ctx->quick = 1;
+            ctx->state = NGX_RESOLVE_NXDOMAIN;
+            ctx->handler(ctx);
+            return NGX_OK;
+        }
      }

      if (ctx->service.len) {
@@ -4652,3 +4682,281 @@

      return p1 - p2;
  }
+
+
+static ngx_int_t
+ngx_resolver_parse_hosts_file(ngx_conf_t *cf, ngx_resolver_t *r)
+{
+    off_t                    file_size;
+    u_char                   ch;
+    u_char                  *start;
+    size_t                   len;
+    ssize_t                  n, size;
+    ngx_int_t                rc;
+    ngx_buf_t                b;
+    ngx_fd_t                 fd;
+    ngx_str_t                filename, s;
+    ngx_file_t               file;
+    in_addr_t                addr;
+    ngx_resolver_node_t     *rn;
+    enum {
+        scan_line = 0,
+        scan_skipline,
+        scan_addr,
+        scan_hosts,
+        scan_name
+    } state;
+
+    b.start = NULL;
+    s.data = NULL;
+    s.len = 0;
+    rn = NULL;
+    rc = NGX_OK;
+
+    filename = r->hosts_file;
+
+    fd = ngx_open_file(filename.data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
+    if (fd == NGX_INVALID_FILE) {
+        ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno,
+                      ngx_open_file_n " \"%s\" failed", filename.data);
+        return NGX_ERROR;
+    }
+
+    ngx_memzero(&file, sizeof(ngx_file_t));
+
+    if (ngx_fd_info(fd, &file.info) == NGX_FILE_ERROR) {
+        ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno,
+                      ngx_fd_info_n " \"%s\" failed", filename.data);
+        goto fail;
+    }
+
+    file.fd = fd;
+    file.log = cf->log;
+    file.name.len = filename.len;
+    file.name.data = filename.data;
+    file.offset = 0;
+
+    b.start = ngx_alloc(NGX_RESOLVER_HOSTSFILE_BUFFER_SIZE, cf->log);
+    if (b.start == NULL) {
+        goto fail;
+    }
+
+    b.pos = b.start;
+    b.last = b.start;
+    b.end = b.last + NGX_RESOLVER_HOSTSFILE_BUFFER_SIZE;
+    b.temporary = 1;
+
+    start = b.pos;
+    state = scan_line;
+    file_size = ngx_file_size(&file.info);
+
+    for ( ;; ) {
+
+        if (b.pos >= b.last) {
+            len = b.pos - start;
+
+            if (len) {
+                ngx_memmove(b.start, start, len);
+            }
+
+            size = (ssize_t) (file_size - file.offset);
+
+            if (size > b.end - (b.start + len)) {
+                size = b.end - (b.start + len);
+
+            } else if (size == 0) {
+                goto done;
+            }
+
+            n = ngx_read_file(&file, b.start + len, size, file.offset);
+            if (n == NGX_ERROR) {
+                ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno,
+                              ngx_read_file_n, " \"%s\" failed",
+                              filename.data);
+                goto fail;
+            }
+
+            if (n != size) {
+                ngx_log_error(NGX_LOG_ERR, cf->log, ngx_errno,
+                              ngx_read_file_n, " returned only %z bytes "
+                              "instead of %z", n, size);
+                goto fail;
+            }
+
+            b.pos = b.start + len;
+            b.last = b.pos + n;
+            start = b.start;
+        }
+
+        ch = *b.pos;
+
+        switch (state) {
+
+        case scan_line:
+            if (ch == ' ') {
+                break;
+            }
+
+            if (ch == '#') {
+                state = scan_skipline;
+                break;
+            }
+
+            if (ch != LF && ch != CR) {
+                start = b.pos;
+                state = scan_addr;
+            }
+
+            break;
+
+        case scan_skipline:
+            if (ch == LF || ch == CR) {
+                state = scan_line;
+            }
+
+            break;
+
+        case scan_addr:
+            if (ch == LF || ch == CR) {
+                state = scan_line;
+                break;
+            }
+
+            if (ch == ' ' || ch == '\t') {
+                if (s.data) {
+                    ngx_free(s.data);
+                }
+
+                s.len = b.pos - start;
+
+                s.data = ngx_alloc(s.len, cf->log);
+                if (s.data == NULL) {
+                    goto fail;
+                }
+
+                ngx_memcpy(s.data, start, s.len);
+
+                state = scan_hosts;
+            }
+
+            break;
+
+        case scan_hosts:
+            if (ch == LF || ch == CR) {
+                state = scan_line;
+                break;
+            }
+
+            if (ch == ' ' || ch == '\t') {
+                break;
+            }
+
+            start = b.pos;
+            state = scan_name;
+            break;
+
+        case scan_name:
+            if (ch == ' ' || ch == '\t' || ch == LF || ch == CR) {
+                rn = ngx_calloc(sizeof(ngx_resolver_node_t), cf->log);
+                if (rn == NULL) {
+                    goto fail;
+                }
+
+                rn->nlen = b.pos - start;
+
+                rn->name = ngx_alloc(rn->nlen, cf->log);
+                if (rn->name == NULL) {
+                    goto fail;
+                }
+
+                ngx_memcpy(rn->name, start, rn->nlen);
+
+                rn->ttl = NGX_MAX_UINT32_VALUE;
+                rn->valid = NGX_MAX_UINT32_VALUE;
+                rn->expire = NGX_MAX_UINT32_VALUE;
+                rn->node.key = ngx_crc32_short(rn->name, rn->nlen);
+
+                if (ngx_strlchr(s.data,
+                                s.data + s.len, ':') != NULL)
+                {
+
+#if (NGX_HAVE_INET6)
+                    if (!r->ipv6
+                        || ngx_inet6_addr(s.data, s.len,
+                                          rn->u6.addr6.s6_addr) != NGX_OK)
+                    {
+#endif
+
+                        ngx_resolver_free_node(r, rn);
+                        state = scan_skipline;
+                        break;
+
+#if (NGX_HAVE_INET6)
+                    }
+
+                    rn->naddrs6 = 1;
+#endif
+
+                } else {
+                    addr = ngx_inet_addr(s.data, s.len);
+                    if (addr == INADDR_NONE) {
+                        ngx_resolver_free_node(r, rn);
+                        state = scan_skipline;
+                        break;
+                    }
+
+                    rn->naddrs = 1;
+                    rn->u.addr = addr;
+                }
+
+                ngx_log_error(NGX_LOG_NOTICE, cf->log, 0,
+                              "host \"%*s\" will resolve to \"%V\" "
+                              "(hosts file at \"%V\")",
+                              rn->nlen, rn->name, &s, &filename);
+
+                ngx_rbtree_insert(&r->name_rbtree, &rn->node);
+
+                ngx_queue_insert_head(&r->name_expire_queue, &rn->queue);
+
+                if (ch == LF || ch == CR) {
+                    state = scan_line;
+                    break;
+                }
+
+                state = scan_hosts;
+            }
+
+            break;
+        }
+
+        b.pos++;
+    }
+
+fail:
+
+    rc = NGX_ERROR;
+
+done:
+
+    if (s.data) {
+        ngx_free(s.data);
+    }
+
+    if (b.start) {
+        ngx_free(b.start);
+    }
+
+    if (ngx_close_file(fd) == NGX_FILE_ERROR) {
+        ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno,
+                      ngx_close_file_n, " \"%s\" failed",
+                      filename.data);
+        rc = NGX_ERROR;
+    }
+
+    if (rc == NGX_ERROR) {
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
diff -r a2f5e25d6a28 -r 558041ef1d70 src/core/ngx_resolver.h
--- a/src/core/ngx_resolver.h    Thu Aug 10 22:21:23 2017 +0300
+++ b/src/core/ngx_resolver.h    Mon Feb 27 19:23:21 2017 -0800
@@ -146,6 +146,8 @@


  struct ngx_resolver_s {
+    ngx_str_t                 hosts_file;
+
      /* has to be pointer because of "incomplete type" */
      ngx_event_t              *event;
      void                     *dummy;

From mdounin at mdounin.ru  Tue Aug 22 13:03:20 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 22 Aug 2017 13:03:20 +0000
Subject: [nginx] SSL: the $ssl_client_escaped_cert variable (ticket #857).
Message-ID: <hg.82f0b8dcca27.1503407000.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/82f0b8dcca27
branches:  
changeset: 7091:82f0b8dcca27
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Tue Aug 22 15:18:10 2017 +0300
description:
SSL: the $ssl_client_escaped_cert variable (ticket #857).

This variable contains URL-encoded client SSL certificate.  In contrast
to $ssl_client_cert, it doesn't depend on deprecated header continuation.
The NGX_ESCAPE_URI_COMPONENT variant of encoding is used, so the resulting
variable can be safely used not only in headers, but also as a request
argument.

The $ssl_client_cert variable should be considered deprecated now.
The $ssl_client_raw_cert variable will be eventually renambed back
to $ssl_client_cert.

diffstat:

 src/event/ngx_event_openssl.c          |  30 ++++++++++++++++++++++++++++++
 src/event/ngx_event_openssl.h          |   2 ++
 src/http/modules/ngx_http_ssl_module.c |   4 ++++
 src/stream/ngx_stream_ssl_module.c     |   4 ++++
 4 files changed, 40 insertions(+), 0 deletions(-)

diffs (80 lines):

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -3663,6 +3663,36 @@ ngx_ssl_get_certificate(ngx_connection_t
 
 
 ngx_int_t
+ngx_ssl_get_escaped_certificate(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s)
+{
+    ngx_str_t  cert;
+    uintptr_t  n;
+
+    if (ngx_ssl_get_raw_certificate(c, pool, &cert) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    if (cert.len == 0) {
+        s->len = 0;
+        return NGX_OK;
+    }
+
+    n = ngx_escape_uri(NULL, cert.data, cert.len, NGX_ESCAPE_URI_COMPONENT);
+
+    s->len = cert.len + n * 2;
+    s->data = ngx_pnalloc(pool, s->len);
+    if (s->data == NULL) {
+        return NGX_ERROR;
+    }
+
+    ngx_escape_uri(s->data, cert.data, cert.len, NGX_ESCAPE_URI_COMPONENT);
+
+    return NGX_OK;
+}
+
+
+ngx_int_t
 ngx_ssl_get_subject_dn(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s)
 {
     BIO        *bio;
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -212,6 +212,8 @@ ngx_int_t ngx_ssl_get_raw_certificate(ng
     ngx_str_t *s);
 ngx_int_t ngx_ssl_get_certificate(ngx_connection_t *c, ngx_pool_t *pool,
     ngx_str_t *s);
+ngx_int_t ngx_ssl_get_escaped_certificate(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
 ngx_int_t ngx_ssl_get_subject_dn(ngx_connection_t *c, ngx_pool_t *pool,
     ngx_str_t *s);
 ngx_int_t ngx_ssl_get_issuer_dn(ngx_connection_t *c, ngx_pool_t *pool,
diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c
--- a/src/http/modules/ngx_http_ssl_module.c
+++ b/src/http/modules/ngx_http_ssl_module.c
@@ -299,6 +299,10 @@ static ngx_http_variable_t  ngx_http_ssl
       (uintptr_t) ngx_ssl_get_raw_certificate,
       NGX_HTTP_VAR_CHANGEABLE, 0 },
 
+    { ngx_string("ssl_client_escaped_cert"), NULL, ngx_http_ssl_variable,
+      (uintptr_t) ngx_ssl_get_escaped_certificate,
+      NGX_HTTP_VAR_CHANGEABLE, 0 },
+
     { ngx_string("ssl_client_s_dn"), NULL, ngx_http_ssl_variable,
       (uintptr_t) ngx_ssl_get_subject_dn, NGX_HTTP_VAR_CHANGEABLE, 0 },
 
diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c
--- a/src/stream/ngx_stream_ssl_module.c
+++ b/src/stream/ngx_stream_ssl_module.c
@@ -249,6 +249,10 @@ static ngx_stream_variable_t  ngx_stream
       (uintptr_t) ngx_ssl_get_raw_certificate,
       NGX_STREAM_VAR_CHANGEABLE, 0 },
 
+    { ngx_string("ssl_client_escaped_cert"), NULL, ngx_stream_ssl_variable,
+      (uintptr_t) ngx_ssl_get_escaped_certificate,
+      NGX_STREAM_VAR_CHANGEABLE, 0 },
+
     { ngx_string("ssl_client_s_dn"), NULL, ngx_stream_ssl_variable,
       (uintptr_t) ngx_ssl_get_subject_dn, NGX_STREAM_VAR_CHANGEABLE, 0 },
 

From mdounin at mdounin.ru  Tue Aug 22 14:40:38 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 22 Aug 2017 14:40:38 +0000
Subject: [nginx] SSL: fixed possible use-after-free in $ssl_server_name.
Message-ID: <hg.2e8de3d81783.1503412838.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/2e8de3d81783
branches:  
changeset: 7092:2e8de3d81783
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Tue Aug 22 17:36:12 2017 +0300
description:
SSL: fixed possible use-after-free in $ssl_server_name.

The $ssl_server_name variable used SSL_get_servername() result directly,
but this is not safe: it references a memory allocation in an SSL
session, and this memory might be freed at any time due to renegotiation.
Instead, copy the name to memory allocated from the pool.

diffstat:

 src/event/ngx_event_openssl.c |  23 ++++++++++++++++-------
 1 files changed, 16 insertions(+), 7 deletions(-)

diffs (33 lines):

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -3551,13 +3551,22 @@ ngx_ssl_get_server_name(ngx_connection_t
 {
 #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
 
-    const char  *servername;
-
-    servername = SSL_get_servername(c->ssl->connection,
-                                    TLSEXT_NAMETYPE_host_name);
-    if (servername) {
-        s->data = (u_char *) servername;
-        s->len = ngx_strlen(servername);
+    size_t       len;
+    const char  *name;
+
+    name = SSL_get_servername(c->ssl->connection, TLSEXT_NAMETYPE_host_name);
+
+    if (name) {
+        len = ngx_strlen(name);
+
+        s->len = len;
+        s->data = ngx_pnalloc(pool, len);
+        if (s->data == NULL) {
+            return NGX_ERROR;
+        }
+
+        ngx_memcpy(s->data, name, len);
+
         return NGX_OK;
     }
 

From fooinha at gmail.com  Tue Aug 22 20:58:32 2017
From: fooinha at gmail.com (Paulo Pacheco)
Date: Tue, 22 Aug 2017 21:58:32 +0100
Subject: [ngx] - save SSL/TLS client hello extensions ids
Message-ID: <CALo6F5Rv6vYoyT+fLL7G5smkOsJRjU=hOLRUSKvLG+xApfi13Q@mail.gmail.com>

Hi,

Is this patch the right way to do it?

My motivation was this: https://github.com/fooinha/nginx-ssl-ja3

Obrigado | Thanx | ???
Paulo Pacheco | ????? ??????


------------------------------------------ CUT HERE
------------------------------------------

diff -r 2e8de3d81783 src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c Tue Aug 22 17:36:12 2017 +0300
+++ b/src/event/ngx_event_openssl.c Tue Aug 22 20:20:30 2017 +0000
@@ -1221,6 +1221,60 @@
 }


+#if OPENSSL_VERSION_NUMBER >= 0x10101000L
+
+int
+ngx_SSL_early_cb_fn(SSL *s, int *al, void *arg) {
+
+    int               got_extensions;
+    int              *ext_out;
+    size_t            ext_len;
+    ngx_connection_t *c;
+
+    c = arg;
+
+    if (c == NULL) {
+        return 1;
+    }
+
+    if (c->ssl == NULL) {
+        return 1;
+    }
+
+    c->ssl->client_extensions_size = 0;
+    c->ssl->client_extensions = NULL;
+
+    got_extensions = SSL_early_get1_extensions_present(s,
+                                                       &ext_out,
+                                                       &ext_len);
+    if (!got_extensions) {
+        return 1;
+    }
+
+    if (!ext_out) {
+        return 1;
+    }
+
+    if (!ext_len) {
+        return 1;
+    }
+
+    c->ssl->client_extensions = ngx_palloc(c->pool, sizeof(int) * ext_len);
+    if (c->ssl->client_extensions == NULL) {
+        OPENSSL_free(ext_out);
+        return 1;
+    }
+
+    c->ssl->client_extensions_size = ext_len;
+    ngx_memcpy(c->ssl->client_extensions, ext_out, sizeof(int) * ext_len);
+
+    OPENSSL_free(ext_out);
+
+    return 1;
+}
+#endif
+
+
 ngx_int_t
 ngx_ssl_handshake(ngx_connection_t *c)
 {
@@ -1229,6 +1283,10 @@

     ngx_ssl_clear_error(c->log);

+#if OPENSSL_VERSION_NUMBER >= 0x10101000L
+    SSL_CTX_set_early_cb(c->ssl->session_ctx, ngx_SSL_early_cb_fn, c);
+#endif
+
     n = SSL_do_handshake(c->ssl->connection);

     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_do_handshake: %d", n);
diff -r 2e8de3d81783 src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h Tue Aug 22 17:36:12 2017 +0300
+++ b/src/event/ngx_event_openssl.h Tue Aug 22 20:20:30 2017 +0000
@@ -85,6 +85,11 @@
     unsigned                    no_wait_shutdown:1;
     unsigned                    no_send_shutdown:1;
     unsigned                    handshake_buffer_set:1;
+
+#if OPENSSL_VERSION_NUMBER >= 0x10101000L
+    size_t                      client_extensions_size;
+    int                        *client_extensions;
+#endif
 };


------------------------------------------ CUT HERE
------------------------------------------

From mdounin at mdounin.ru  Wed Aug 23 16:11:12 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Wed, 23 Aug 2017 19:11:12 +0300
Subject: [ngx] - save SSL/TLS client hello extensions ids
In-Reply-To: <CALo6F5Rv6vYoyT+fLL7G5smkOsJRjU=hOLRUSKvLG+xApfi13Q@mail.gmail.com>
References: <CALo6F5Rv6vYoyT+fLL7G5smkOsJRjU=hOLRUSKvLG+xApfi13Q@mail.gmail.com>
Message-ID: <20170823161112.GO93611@mdounin.ru>

Hello!

On Tue, Aug 22, 2017 at 09:58:32PM +0100, Paulo Pacheco wrote:

> Hi,
> 
> Is this patch the right way to do it?
> 
> My motivation was this: https://github.com/fooinha/nginx-ssl-ja3

Saving the list of TLS extensions for future use just in case we'll 
need it in some 3rd-party module looks suboptimal, so please don't 
expected this patch to be merged.

[...]

> @@ -1229,6 +1283,10 @@
> 
>      ngx_ssl_clear_error(c->log);
> 
> +#if OPENSSL_VERSION_NUMBER >= 0x10101000L
> +    SSL_CTX_set_early_cb(c->ssl->session_ctx, ngx_SSL_early_cb_fn, c);
> +#endif
> +

Modifications of SSL contexts should be done during configuration, 
not at run-time.  Instead, you should do this when a context is 
configured, somewhere in ngx_http_ssl_merge_srv_conf() after the 
ngx_ssl_create() call.

Note well that you can install callback in your own module - this 
will be a hack and the code will break if/when nginx will start 
using SSL_CTX_set_early_cb() for some reason, but this will allow 
your module to work for now without any modifications of nginx 
core.  This approach should be good enough at least for 
experiments.

-- 
Maxim Dounin
http://nginx.org/

From mdounin at mdounin.ru  Wed Aug 23 16:20:31 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Wed, 23 Aug 2017 16:20:31 +0000
Subject: [nginx] Upstream: unconditional parsing of last_modified_time.
Message-ID: <hg.acc2cddc7b45.1503505231.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/acc2cddc7b45
branches:  
changeset: 7093:acc2cddc7b45
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Wed Aug 23 19:20:06 2017 +0300
description:
Upstream: unconditional parsing of last_modified_time.

This fixes at least the following cases, where no last_modified_time
(assuming caching is not enabled) resulted in incorrect behaviour:

- slice filter and If-Range requests (ticket #1357);
- If-Range requests with proxy_force_ranges;
- expires modified.

diffstat:

 src/http/ngx_http_upstream.c |  20 +++-----------------
 1 files changed, 3 insertions(+), 17 deletions(-)

diffs (38 lines):

diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -4390,15 +4390,8 @@ ngx_http_upstream_process_last_modified(
     u = r->upstream;
 
     u->headers_in.last_modified = h;
-
-#if (NGX_HTTP_CACHE)
-
-    if (u->cacheable) {
-        u->headers_in.last_modified_time = ngx_parse_http_time(h->value.data,
-                                                               h->value.len);
-    }
-
-#endif
+    u->headers_in.last_modified_time = ngx_parse_http_time(h->value.data,
+                                                           h->value.len);
 
     return NGX_OK;
 }
@@ -4940,15 +4933,8 @@ ngx_http_upstream_copy_last_modified(ngx
     *ho = *h;
 
     r->headers_out.last_modified = ho;
-
-#if (NGX_HTTP_CACHE)
-
-    if (r->upstream->cacheable) {
-        r->headers_out.last_modified_time =
+    r->headers_out.last_modified_time =
                                     r->upstream->headers_in.last_modified_time;
-    }
-
-#endif
 
     return NGX_OK;
 }

From arut at nginx.com  Wed Aug 23 17:59:39 2017
From: arut at nginx.com (Roman Arutyunyan)
Date: Wed, 23 Aug 2017 17:59:39 +0000
Subject: [nginx] Secure link: fixed stack buffer overflow.
Message-ID: <hg.c7d4017c8876.1503511179.6026610855610030274@dev.nginx.com>

details:   http://hg.nginx.org/nginx/rev/c7d4017c8876
branches:  
changeset: 7094:c7d4017c8876
user:      Roman Arutyunyan <arut at nginx.com>
date:      Tue Aug 22 21:22:59 2017 +0300
description:
Secure link: fixed stack buffer overflow.

When secure link checksum has length of 23 or 24 bytes, decoded base64 value
could occupy 17 or 18 bytes which is more than 16 bytes previously allocated
for it on stack.  The buffer overflow does not have any security implications
since only one local variable was corrupted and this variable was not used in
this case.

The fix is to increase buffer size up to 18 bytes.  Useless buffer size
initialization is removed as well.

diffstat:

 src/http/modules/ngx_http_secure_link_module.c |  3 +--
 1 files changed, 1 insertions(+), 2 deletions(-)

diffs (20 lines):

diff -r acc2cddc7b45 -r c7d4017c8876 src/http/modules/ngx_http_secure_link_module.c
--- a/src/http/modules/ngx_http_secure_link_module.c	Wed Aug 23 19:20:06 2017 +0300
+++ b/src/http/modules/ngx_http_secure_link_module.c	Tue Aug 22 21:22:59 2017 +0300
@@ -107,7 +107,7 @@ ngx_http_secure_link_variable(ngx_http_r
     ngx_md5_t                     md5;
     ngx_http_secure_link_ctx_t   *ctx;
     ngx_http_secure_link_conf_t  *conf;
-    u_char                        hash_buf[16], md5_buf[16];
+    u_char                        hash_buf[18], md5_buf[16];
 
     conf = ngx_http_get_module_loc_conf(r, ngx_http_secure_link_module);
 
@@ -154,7 +154,6 @@ ngx_http_secure_link_variable(ngx_http_r
         goto not_found;
     }
 
-    hash.len = 16;
     hash.data = hash_buf;
 
     if (ngx_decode_base64url(&hash, &val) != NGX_OK) {

From nate.karstens at garmin.com  Thu Aug 24 02:19:39 2017
From: nate.karstens at garmin.com (Nate Karstens)
Date: Wed, 23 Aug 2017 21:19:39 -0500
Subject: [PATCH] [PATCH 1 of 4] Core: add function to decode hexadecimal
 strings
In-Reply-To: <010418>
References: <010418>
Message-ID: <17c038b56051f922ec44.1503541179@OLA-6J5NNS1.ad.garmin.com>

# HG changeset patch
# User Nate Karstens <nate.karstens at garmin.com>
# Date 1503540018 18000
#      Wed Aug 23 21:00:18 2017 -0500
# Node ID 17c038b56051f922ec440d40e23e8d1b23d835df
# Parent  c7d4017c8876af6d8570e400320537d7d39e9578
[PATCH 1 of 4] Core: add function to decode hexadecimal strings.

Adds functionality to convert a hexadecimal string into binary data.
This will be used to decode PSKs stored in hexadecimal representation.

Signed-off-by: Nate Karstens <nate.karstens at garmin.com>

diff -r c7d4017c8876 -r 17c038b56051 src/core/ngx_string.c
--- a/src/core/ngx_string.c     Tue Aug 22 21:22:59 2017 +0300
+++ b/src/core/ngx_string.c     Wed Aug 23 21:00:18 2017 -0500
@@ -1118,6 +1118,57 @@ ngx_hex_dump(u_char *dst, u_char *src, s
 }


+ngx_int_t
+ngx_hex_decode(u_char *dst, u_char *src, size_t len)
+{
+    u_char     ch, decoded;
+
+    if (len & 1) {
+        return NGX_ERROR;
+    }
+
+    while (len > 0) {
+        ch = *src++;
+        len--;
+
+        if (len & 1) {
+
+            if (ch >= '0' && ch <= '9') {
+                decoded = ch - '0';
+                continue;
+            }
+
+            ch |= 0x20;
+
+            if (ch >= 'a' && ch <= 'f') {
+                decoded = ch - 'a' + 10;
+                continue;
+            }
+
+            return NGX_ERROR;
+
+        } else {
+
+            if (ch >= '0' && ch <= '9') {
+                *dst++ = (u_char) ((decoded << 4) + ch - '0');
+                continue;
+            }
+
+            ch |= 0x20;
+
+            if (ch >= 'a' && ch <= 'f') {
+                *dst++ = (u_char) ((decoded << 4) + ch - 'a' + 10);
+                continue;
+            }
+
+            return NGX_ERROR;
+        }
+    }
+
+    return NGX_OK;
+}
+
+
 void
 ngx_encode_base64(ngx_str_t *dst, ngx_str_t *src)
 {
diff -r c7d4017c8876 -r 17c038b56051 src/core/ngx_string.h
--- a/src/core/ngx_string.h     Tue Aug 22 21:22:59 2017 +0300
+++ b/src/core/ngx_string.h     Wed Aug 23 21:00:18 2017 -0500
@@ -177,6 +177,7 @@ time_t ngx_atotm(u_char *line, size_t n)
 ngx_int_t ngx_hextoi(u_char *line, size_t n);

 u_char *ngx_hex_dump(u_char *dst, u_char *src, size_t len);
+ngx_int_t ngx_hex_decode(u_char *dst, u_char *src, size_t len);


 #define ngx_base64_encoded_length(len)  (((len + 2) / 3) * 4)

________________________________

CONFIDENTIALITY NOTICE: This email and any attachments are for the sole use of the intended recipient(s) and contain information that may be Garmin confidential and/or Garmin legally privileged. If you have received this email in error, please notify the sender by reply email and delete the message. Any disclosure, copying, distribution or use of this communication (including attachments) by someone other than the intended recipient is prohibited. Thank you.

From nate.karstens at garmin.com  Thu Aug 24 02:20:50 2017
From: nate.karstens at garmin.com (Nate Karstens)
Date: Wed, 23 Aug 2017 21:20:50 -0500
Subject: [PATCH] [PATCH 2 of 4] SSL: add support for PSK cipher suites
In-Reply-To: <010419>
References: <010419>
Message-ID: <97953fe374455a049732.1503541250@OLA-6J5NNS1.ad.garmin.com>

# HG changeset patch
# User Nate Karstens <nate.karstens at garmin.com>
# Date 1503540059 18000
#      Wed Aug 23 21:00:59 2017 -0500
# Node ID 97953fe374455a04973268c4b2fbadd7ced91ffe
# Parent  17c038b56051f922ec440d40e23e8d1b23d835df
[PATCH 2 of 4] SSL: add support for PSK cipher suites.

Adds support for TLS connections using PSK cipher suites. A new
configuration directive, ssl_psk_file, specifies the file that
contains a list of identities and associated PSKs. Each line of
the file begins with the identity, followed by a colon character
(':'), and ending with the PSK. As required by RFC 4279 section
5.4, PSKs may be entered either as plain text or using hexadecimal
encoding. Hexadecimal PSKs must begin with "{HEX}". PSKs without
this prefix are assumed to be plain text, but they may optionally
begin with "{PLAIN}" to denote this. Some examples:

gary:plain_text_password
min:{PLAIN}another_text_password
cliff:{HEX}ab0123CD

PSK functionality can be easily tested with the OpenSSL s_client
using the "-psk" and "-psk_identity" options.

Signed-off-by: Nate Karstens <nate.karstens at garmin.com>

diff -r 17c038b56051 -r 97953fe37445 contrib/vim/syntax/nginx.vim
--- a/contrib/vim/syntax/nginx.vim      Wed Aug 23 21:00:18 2017 -0500
+++ b/contrib/vim/syntax/nginx.vim      Wed Aug 23 21:00:59 2017 -0500
@@ -550,6 +550,7 @@ syn keyword ngxDirective contained ssl_p
 syn keyword ngxDirective contained ssl_prefer_server_ciphers
 syn keyword ngxDirective contained ssl_preread
 syn keyword ngxDirective contained ssl_protocols
+syn keyword ngxDirective contained ssl_psk_file
 syn keyword ngxDirective contained ssl_session_cache
 syn keyword ngxDirective contained ssl_session_ticket_key
 syn keyword ngxDirective contained ssl_session_tickets
diff -r 17c038b56051 -r 97953fe37445 src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c     Wed Aug 23 21:00:18 2017 -0500
+++ b/src/event/ngx_event_openssl.c     Wed Aug 23 21:00:59 2017 -0500
@@ -11,6 +11,7 @@


 #define NGX_SSL_PASSWORD_BUFFER_SIZE  4096
+#define NGX_SSL_PSK_BUFFER_SIZE       4096


 typedef struct {
@@ -24,6 +25,10 @@ static int ngx_ssl_verify_callback(int o
 static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where,
     int ret);
 static void ngx_ssl_passwords_cleanup(void *data);
+#ifdef PSK_MAX_IDENTITY_LEN
+static unsigned int ngx_ssl_psk_callback(ngx_ssl_conn_t *ssl_conn,
+    const char *identity, unsigned char *psk, unsigned int max_psk_len);
+#endif
 static void ngx_ssl_handshake_handler(ngx_event_t *ev);
 static ngx_int_t ngx_ssl_handle_recv(ngx_connection_t *c, int n);
 static void ngx_ssl_write_handler(ngx_event_t *wev);
@@ -110,6 +115,7 @@ int  ngx_ssl_connection_index;
 int  ngx_ssl_server_conf_index;
 int  ngx_ssl_session_cache_index;
 int  ngx_ssl_session_ticket_keys_index;
+int  ngx_ssl_psk_index;
 int  ngx_ssl_certificate_index;
 int  ngx_ssl_next_certificate_index;
 int  ngx_ssl_certificate_name_index;
@@ -195,6 +201,14 @@ ngx_ssl_init(ngx_log_t *log)
         return NGX_ERROR;
     }

+    ngx_ssl_psk_index = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL);
+
+    if (ngx_ssl_psk_index == -1) {
+        ngx_ssl_error(NGX_LOG_ALERT, log, 0,
+                      "SSL_CTX_get_ex_new_index() failed");
+        return NGX_ERROR;
+    }
+
     ngx_ssl_certificate_index = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL,
                                                          NULL);
     if (ngx_ssl_certificate_index == -1) {
@@ -1163,6 +1177,211 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_s


 ngx_int_t
+ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file)
+{
+#ifdef PSK_MAX_IDENTITY_LEN
+
+    ngx_fd_t            fd;
+    ngx_int_t           err;
+    ngx_file_info_t     fi;
+
+    err = NGX_OK;
+
+    if (ngx_conf_full_name(cf->cycle, file, 1) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    fd = ngx_open_file(file->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
+    if (fd == NGX_INVALID_FILE) {
+        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, ngx_errno,
+                      ngx_open_file_n " \"%V\" failed", file);
+        return NGX_ERROR;
+    }
+
+    if (ngx_fd_info(fd, &fi) == NGX_FILE_ERROR) {
+        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, ngx_errno,
+                      ngx_fd_info_n " \"%V\" failed", file);
+        err = NGX_ERROR;
+        goto failed;
+    }
+
+    if (ngx_file_size(&fi) == 0) {
+        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
+                      "PSK file \"%V\" is empty", file);
+        err = NGX_ERROR;
+        goto failed;
+    }
+
+failed:
+
+    if (ngx_close_file(fd) == NGX_FILE_ERROR) {
+        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, ngx_errno,
+                      ngx_close_file_n " %V failed", file);
+        err = NGX_ERROR;
+    }
+
+    if (err != NGX_OK) {
+        return err;
+    }
+
+    if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_psk_index, file) == 0) {
+        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
+                      "SSL_CTX_set_ex_data() failed");
+        return NGX_ERROR;
+    }
+
+    SSL_CTX_set_psk_server_callback(ssl->ctx, ngx_ssl_psk_callback);
+
+#endif
+
+    return NGX_OK;
+}
+
+
+#ifdef PSK_MAX_IDENTITY_LEN
+
+static unsigned int
+ngx_ssl_psk_callback(ngx_ssl_conn_t *ssl_conn, const char *identity,
+    unsigned char *psk, unsigned int max_psk_len)
+{
+    u_char             *p, *last, *end, *colon;
+    size_t              len;
+    ssize_t             n;
+    SSL_CTX            *ssl_ctx;
+    ngx_fd_t            fd;
+    ngx_str_t          *file;
+    unsigned int        psk_len;
+    ngx_connection_t   *c;
+    u_char              buf[NGX_SSL_PSK_BUFFER_SIZE];
+
+    psk_len = 0;
+    c = ngx_ssl_get_connection(ssl_conn);
+
+    ssl_ctx = SSL_get_SSL_CTX(ssl_conn);
+    file = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_psk_index);
+
+    fd = ngx_open_file(file->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
+    if (fd == NGX_INVALID_FILE) {
+        ngx_ssl_error(NGX_LOG_ERR, c->log, ngx_errno,
+                      ngx_open_file_n " \"%V\" failed", file);
+        return 0;
+    }
+
+    len = 0;
+    last = buf;
+
+    do {
+        n = ngx_read_fd(fd, last, NGX_SSL_PSK_BUFFER_SIZE - len);
+
+        if (n == -1) {
+            ngx_ssl_error(NGX_LOG_ERR, c->log, ngx_errno,
+                          ngx_read_fd_n " \"%V\" failed", file);
+            psk_len = 0;
+            goto cleanup;
+        }
+
+        end = last + n;
+
+        if (len && n == 0) {
+            *end++ = LF;
+        }
+
+        for (p = buf; /* void */; p = last) {
+            last = ngx_strlchr(last, end, LF);
+
+            if (last == NULL) {
+                break;
+            }
+
+            len = last++ - p;
+
+            if (len && p[len - 1] == CR) {
+                len--;
+            }
+
+            if (len == 0) {
+                continue;
+            }
+
+            colon = ngx_strlchr(p, p + len, ':');
+
+            if (colon == NULL) {
+                continue;
+            }
+
+            *colon = '\0';
+
+            if (ngx_strcmp(p, identity) != 0) {
+                continue;
+            }
+
+            len -= colon + 1 - p;
+            p = colon + 1;
+
+            if (ngx_strncmp(p, "{HEX}", sizeof("{HEX}") - 1) == 0) {
+
+                p += sizeof("{HEX}") - 1;
+                len -= sizeof("{HEX}") - 1;
+
+                if (len / 2 > max_psk_len) {
+                    goto cleanup;
+                }
+
+                if (ngx_hex_decode(psk, p, len) != NGX_OK) {
+                    ngx_memzero(psk, len / 2);
+                    goto cleanup;
+                }
+
+                psk_len = len / 2;
+
+                goto cleanup;
+
+            } else if (ngx_strncmp(p, "{PLAIN}", sizeof("{PLAIN}") - 1) == 0) {
+                p += sizeof("{PLAIN}") - 1;
+                len -= sizeof("{PLAIN}") - 1;
+            }
+
+            if (len > max_psk_len) {
+                goto cleanup;
+            }
+
+            ngx_memcpy(psk, p, len);
+            psk_len = len;
+
+            goto cleanup;
+        }
+
+        len = end - p;
+
+        if (len == NGX_SSL_PSK_BUFFER_SIZE) {
+            ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
+                          "too long line in \"%V\"", file);
+            psk_len = 0;
+            goto cleanup;
+        }
+
+        ngx_memmove(buf, p, len);
+        last = buf + len;
+
+    } while (n != 0);
+
+cleanup:
+
+    if (ngx_close_file(fd) == NGX_FILE_ERROR) {
+        ngx_ssl_error(NGX_LOG_ALERT, c->log, ngx_errno,
+                      ngx_close_file_n " %V failed", file);
+        psk_len = 0;
+    }
+
+    ngx_memzero(buf, NGX_SSL_PSK_BUFFER_SIZE);
+
+    return psk_len;
+}
+
+#endif
+
+
+ngx_int_t
 ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags)
 {
     ngx_ssl_connection_t  *sc;
diff -r 17c038b56051 -r 97953fe37445 src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h     Wed Aug 23 21:00:18 2017 -0500
+++ b/src/event/ngx_event_openssl.h     Wed Aug 23 21:00:59 2017 -0500
@@ -172,6 +172,7 @@ ngx_int_t ngx_ssl_session_cache(ngx_ssl_
 ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl,
     ngx_array_t *paths);
 ngx_int_t ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data);
+ngx_int_t ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
 ngx_int_t ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c,
     ngx_uint_t flags);

@@ -253,6 +254,7 @@ extern int  ngx_ssl_connection_index;
 extern int  ngx_ssl_server_conf_index;
 extern int  ngx_ssl_session_cache_index;
 extern int  ngx_ssl_session_ticket_keys_index;
+extern int  ngx_ssl_psk_index;
 extern int  ngx_ssl_certificate_index;
 extern int  ngx_ssl_next_certificate_index;
 extern int  ngx_ssl_certificate_name_index;
diff -r 17c038b56051 -r 97953fe37445 src/http/modules/ngx_http_ssl_module.c
--- a/src/http/modules/ngx_http_ssl_module.c    Wed Aug 23 21:00:18 2017 -0500
+++ b/src/http/modules/ngx_http_ssl_module.c    Wed Aug 23 21:00:59 2017 -0500
@@ -234,6 +234,13 @@ static ngx_command_t  ngx_http_ssl_comma
       offsetof(ngx_http_ssl_srv_conf_t, stapling_verify),
       NULL },

+    { ngx_string("ssl_psk_file"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_str_slot,
+      NGX_HTTP_SRV_CONF_OFFSET,
+      offsetof(ngx_http_ssl_srv_conf_t, psk_file),
+      NULL },
+
       ngx_null_command
 };

@@ -543,6 +550,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t
      *     sscf->shm_zone = NULL;
      *     sscf->stapling_file = { 0, NULL };
      *     sscf->stapling_responder = { 0, NULL };
+     *     sscf->psk_file = { 0, NULL };
      */

     sscf->enable = NGX_CONF_UNSET;
@@ -624,6 +632,8 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *
     ngx_conf_merge_str_value(conf->stapling_responder,
                          prev->stapling_responder, "");

+    ngx_conf_merge_str_value(conf->psk_file, prev->psk_file, "");
+
     conf->ssl.log = cf->log;

     if (conf->enable) {
@@ -804,6 +814,10 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *

     }

+    if (ngx_ssl_psk_file(cf, &conf->ssl, &conf->psk_file) != NGX_OK) {
+        return NGX_CONF_ERROR;
+    }
+
     return NGX_CONF_OK;
 }

diff -r 17c038b56051 -r 97953fe37445 src/http/modules/ngx_http_ssl_module.h
--- a/src/http/modules/ngx_http_ssl_module.h    Wed Aug 23 21:00:18 2017 -0500
+++ b/src/http/modules/ngx_http_ssl_module.h    Wed Aug 23 21:00:59 2017 -0500
@@ -55,6 +55,8 @@ typedef struct {
     ngx_str_t                       stapling_file;
     ngx_str_t                       stapling_responder;

+    ngx_str_t                       psk_file;
+
     u_char                         *file;
     ngx_uint_t                      line;
 } ngx_http_ssl_srv_conf_t;

________________________________

CONFIDENTIALITY NOTICE: This email and any attachments are for the sole use of the intended recipient(s) and contain information that may be Garmin confidential and/or Garmin legally privileged. If you have received this email in error, please notify the sender by reply email and delete the message. Any disclosure, copying, distribution or use of this communication (including attachments) by someone other than the intended recipient is prohibited. Thank you.

From nate.karstens at garmin.com  Thu Aug 24 02:22:17 2017
From: nate.karstens at garmin.com (Nate Karstens)
Date: Wed, 23 Aug 2017 21:22:17 -0500
Subject: [PATCH] [PATCH 3 of 4] SSL: add PSK identity variable
In-Reply-To: <010420>
References: <010420>
Message-ID: <a11e114a2bcde4afb515.1503541337@OLA-6J5NNS1.ad.garmin.com>

# HG changeset patch
# User Nate Karstens <nate.karstens at garmin.com>
# Date 1503540211 18000
#      Wed Aug 23 21:03:31 2017 -0500
# Node ID a11e114a2bcde4afb515dd0b70f3ef39693f475a
# Parent  97953fe374455a04973268c4b2fbadd7ced91ffe
[PATCH 3 of 4] SSL: add PSK identity variable.

Adds the variable $ssl_psk_identity to get the PSK identity
used in a connnection secured with a PSK cipher suite.

Signed-off-by: Nate Karstens <nate.karstens at garmin.com>

diff -r 97953fe37445 -r a11e114a2bcd src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c     Wed Aug 23 21:00:59 2017 -0500
+++ b/src/event/ngx_event_openssl.c     Wed Aug 23 21:03:31 2017 -0500
@@ -4388,6 +4388,38 @@ ngx_ssl_parse_time(
 }


+ngx_int_t
+ngx_ssl_get_psk_identity(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s)
+{
+#ifdef PSK_MAX_IDENTITY_LEN
+
+    const char  *identity;
+
+    s->len = 0;
+
+    identity = SSL_get_psk_identity(c->ssl->connection);
+
+    if (identity) {
+        s->len = ngx_strlen(identity);
+
+        s->data = ngx_pnalloc(pool, s->len + 1);
+        if (s->data == NULL) {
+            return NGX_ERROR;
+        }
+
+        ngx_cpystrn(s->data, (u_char *) identity, s->len + 1);
+    }
+
+#else
+
+    s->len = 0;
+
+#endif
+
+    return NGX_OK;
+}
+
+
 static void *
 ngx_openssl_create_conf(ngx_cycle_t *cycle)
 {
diff -r 97953fe37445 -r a11e114a2bcd src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h     Wed Aug 23 21:00:59 2017 -0500
+++ b/src/event/ngx_event_openssl.h     Wed Aug 23 21:03:31 2017 -0500
@@ -235,6 +235,8 @@ ngx_int_t ngx_ssl_get_client_v_end(ngx_c
     ngx_str_t *s);
 ngx_int_t ngx_ssl_get_client_v_remain(ngx_connection_t *c, ngx_pool_t *pool,
     ngx_str_t *s);
+ngx_int_t ngx_ssl_get_psk_identity(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);


 ngx_int_t ngx_ssl_handshake(ngx_connection_t *c);
diff -r 97953fe37445 -r a11e114a2bcd src/http/modules/ngx_http_ssl_module.c
--- a/src/http/modules/ngx_http_ssl_module.c    Wed Aug 23 21:00:59 2017 -0500
+++ b/src/http/modules/ngx_http_ssl_module.c    Wed Aug 23 21:03:31 2017 -0500
@@ -340,6 +340,9 @@ static ngx_http_variable_t  ngx_http_ssl
     { ngx_string("ssl_client_v_remain"), NULL, ngx_http_ssl_variable,
       (uintptr_t) ngx_ssl_get_client_v_remain, NGX_HTTP_VAR_CHANGEABLE, 0 },

+    { ngx_string("ssl_psk_identity"), NULL, ngx_http_ssl_variable,
+      (uintptr_t) ngx_ssl_get_psk_identity, NGX_HTTP_VAR_CHANGEABLE, 0 },
+
       ngx_http_null_variable
 };


________________________________

CONFIDENTIALITY NOTICE: This email and any attachments are for the sole use of the intended recipient(s) and contain information that may be Garmin confidential and/or Garmin legally privileged. If you have received this email in error, please notify the sender by reply email and delete the message. Any disclosure, copying, distribution or use of this communication (including attachments) by someone other than the intended recipient is prohibited. Thank you.

From nate.karstens at garmin.com  Thu Aug 24 02:22:43 2017
From: nate.karstens at garmin.com (Nate Karstens)
Date: Wed, 23 Aug 2017 21:22:43 -0500
Subject: [PATCH] [PATCH 4 of 4] SSL: add identity hint config directive
In-Reply-To: <010421>
References: <010421>
Message-ID: <62b4032371bd45217d40.1503541363@OLA-6J5NNS1.ad.garmin.com>

# HG changeset patch
# User Nate Karstens <nate.karstens at garmin.com>
# Date 1503540237 18000
#      Wed Aug 23 21:03:57 2017 -0500
# Node ID 62b4032371bd45217d40e2f0daf8ecd6956601d8
# Parent  a11e114a2bcde4afb515dd0b70f3ef39693f475a
[PATCH 4 of 4] SSL: add identity hint config directive.

Adds the directive "ssl_psk_identity_hint" to the ngx_http_ssl_module.
This allows the user to specify the PSK identity hint given to the
connecting client.

Signed-off-by: Nate Karstens <nate.karstens at garmin.com>

diff -r a11e114a2bcd -r 62b4032371bd contrib/vim/syntax/nginx.vim
--- a/contrib/vim/syntax/nginx.vim      Wed Aug 23 21:03:31 2017 -0500
+++ b/contrib/vim/syntax/nginx.vim      Wed Aug 23 21:03:57 2017 -0500
@@ -551,6 +551,7 @@ syn keyword ngxDirective contained ssl_p
 syn keyword ngxDirective contained ssl_preread
 syn keyword ngxDirective contained ssl_protocols
 syn keyword ngxDirective contained ssl_psk_file
+syn keyword ngxDirective contained ssl_psk_identity_hint
 syn keyword ngxDirective contained ssl_session_cache
 syn keyword ngxDirective contained ssl_session_ticket_key
 syn keyword ngxDirective contained ssl_session_tickets
diff -r a11e114a2bcd -r 62b4032371bd src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c     Wed Aug 23 21:03:31 2017 -0500
+++ b/src/event/ngx_event_openssl.c     Wed Aug 23 21:03:57 2017 -0500
@@ -1177,7 +1177,8 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_s


 ngx_int_t
-ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file)
+ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file,
+    ngx_str_t *identity_hint)
 {
 #ifdef PSK_MAX_IDENTITY_LEN

@@ -1230,6 +1231,14 @@ failed:
         return NGX_ERROR;
     }

+    if (SSL_CTX_use_psk_identity_hint(ssl->ctx, (char *) identity_hint->data)
+        == 0)
+    {
+        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
+                      "SSL_CTX_use_psk_identity_hint() failed");
+        return NGX_ERROR;
+    }
+
     SSL_CTX_set_psk_server_callback(ssl->ctx, ngx_ssl_psk_callback);

 #endif
diff -r a11e114a2bcd -r 62b4032371bd src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h     Wed Aug 23 21:03:31 2017 -0500
+++ b/src/event/ngx_event_openssl.h     Wed Aug 23 21:03:57 2017 -0500
@@ -172,7 +172,8 @@ ngx_int_t ngx_ssl_session_cache(ngx_ssl_
 ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl,
     ngx_array_t *paths);
 ngx_int_t ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data);
-ngx_int_t ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
+ngx_int_t ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file,
+    ngx_str_t *identity_hint);
 ngx_int_t ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c,
     ngx_uint_t flags);

diff -r a11e114a2bcd -r 62b4032371bd src/http/modules/ngx_http_ssl_module.c
--- a/src/http/modules/ngx_http_ssl_module.c    Wed Aug 23 21:03:31 2017 -0500
+++ b/src/http/modules/ngx_http_ssl_module.c    Wed Aug 23 21:03:57 2017 -0500
@@ -241,6 +241,13 @@ static ngx_command_t  ngx_http_ssl_comma
       offsetof(ngx_http_ssl_srv_conf_t, psk_file),
       NULL },

+    { ngx_string("ssl_psk_identity_hint"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_str_slot,
+      NGX_HTTP_SRV_CONF_OFFSET,
+      offsetof(ngx_http_ssl_srv_conf_t, psk_identity_hint),
+      NULL },
+
       ngx_null_command
 };

@@ -554,6 +561,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t
      *     sscf->stapling_file = { 0, NULL };
      *     sscf->stapling_responder = { 0, NULL };
      *     sscf->psk_file = { 0, NULL };
+     *     sscf->psk_identity_hint = { 0, NULL };
      */

     sscf->enable = NGX_CONF_UNSET;
@@ -636,6 +644,8 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *
                          prev->stapling_responder, "");

     ngx_conf_merge_str_value(conf->psk_file, prev->psk_file, "");
+    ngx_conf_merge_str_value(conf->psk_identity_hint,
+                         prev->psk_identity_hint, "");

     conf->ssl.log = cf->log;

@@ -817,7 +827,10 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *

     }

-    if (ngx_ssl_psk_file(cf, &conf->ssl, &conf->psk_file) != NGX_OK) {
+    if (ngx_ssl_psk_file(cf, &conf->ssl, &conf->psk_file,
+                         &conf->psk_identity_hint)
+        != NGX_OK)
+    {
         return NGX_CONF_ERROR;
     }

diff -r a11e114a2bcd -r 62b4032371bd src/http/modules/ngx_http_ssl_module.h
--- a/src/http/modules/ngx_http_ssl_module.h    Wed Aug 23 21:03:31 2017 -0500
+++ b/src/http/modules/ngx_http_ssl_module.h    Wed Aug 23 21:03:57 2017 -0500
@@ -56,6 +56,7 @@ typedef struct {
     ngx_str_t                       stapling_responder;

     ngx_str_t                       psk_file;
+    ngx_str_t                       psk_identity_hint;

     u_char                         *file;
     ngx_uint_t                      line;

________________________________

CONFIDENTIALITY NOTICE: This email and any attachments are for the sole use of the intended recipient(s) and contain information that may be Garmin confidential and/or Garmin legally privileged. If you have received this email in error, please notify the sender by reply email and delete the message. Any disclosure, copying, distribution or use of this communication (including attachments) by someone other than the intended recipient is prohibited. Thank you.

From pluknet at nginx.com  Thu Aug 24 13:21:09 2017
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Thu, 24 Aug 2017 13:21:09 +0000
Subject: [njs] Fixed building by GCC 7 with -Wimplicit-fallthrough.
Message-ID: <hg.746c49e6c195.1503580869.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/746c49e6c195
branches:  
changeset: 399:746c49e6c195
user:      Sergey Kandaurov <pluknet at nginx.com>
date:      Thu Aug 24 16:16:37 2017 +0300
description:
Fixed building by GCC 7 with -Wimplicit-fallthrough.

diffstat:

 nxt/nxt_murmur_hash.c       |  2 ++
 nxt/test/lvlhsh_unit_test.c |  1 +
 2 files changed, 3 insertions(+), 0 deletions(-)

diffs (25 lines):

diff -r 4b11d945ce7d -r 746c49e6c195 nxt/nxt_murmur_hash.c
--- a/nxt/nxt_murmur_hash.c	Fri Aug 18 18:40:39 2017 +0300
+++ b/nxt/nxt_murmur_hash.c	Thu Aug 24 16:16:37 2017 +0300
@@ -40,8 +40,10 @@ nxt_murmur_hash2(const void *data, size_
     switch (len) {
     case 3:
         h ^= p[2] << 16;
+        /* Fall through. */
     case 2:
         h ^= p[1] << 8;
+        /* Fall through. */
     case 1:
         h ^= p[0];
         h *= m;
diff -r 4b11d945ce7d -r 746c49e6c195 nxt/test/lvlhsh_unit_test.c
--- a/nxt/test/lvlhsh_unit_test.c	Fri Aug 18 18:40:39 2017 +0300
+++ b/nxt/test/lvlhsh_unit_test.c	Thu Aug 24 16:16:37 2017 +0300
@@ -74,6 +74,7 @@ lvlhsh_unit_test_add(nxt_lvlhsh_t *lh, c
     case NXT_DECLINED:
         printf("lvlhsh unit test failed: key %08lX is already in hash\n",
                (long) key);
+        /* Fall through. */
 
     default:
         return NXT_ERROR;

From jsdevel at kogo.io  Mon Aug 28 22:26:21 2017
From: jsdevel at kogo.io (Joseph Spencer)
Date: Mon, 28 Aug 2017 15:26:21 -0700
Subject: Simple rewrite handler
Message-ID: <CABPA3Sj8q_u0j3zjDLooHyVHwtB_C-x+xh1O9NjbJ81fApAwbQ@mail.gmail.com>

Hello,

I've been pouring over documentation and source code for the rewrite
module, but I'm having difficulty understanding how I could go about
implementing a simple re-write handler.  I have a few questions and I'm
hoping that someone can answer:

   - Does the handler have to be registered in one of the rewrite phases?
   - Is it necessary to use captures?  Or can I just modify r->uri and
   return the proper status?


Thanks,
Joe
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20170828/e33f56c8/attachment.html>

From xeioex at nginx.com  Tue Aug 29 11:29:01 2017
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Tue, 29 Aug 2017 11:29:01 +0000
Subject: [njs] Introduced function level backtrace.
Message-ID: <hg.82dc332cb4c8.1504006141.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/82dc332cb4c8
branches:  
changeset: 400:82dc332cb4c8
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Aug 29 14:06:23 2017 +0300
description:
Introduced function level backtrace.

diffstat:

 njs/njs.c                       |   40 +++++++++++-
 njs/njs_builtin.c               |  133 ++++++++++++++++++++++++++++++++++++++++
 njs/njs_generator.c             |   48 ++++++++++++++-
 njs/njs_parser.c                |    2 +
 njs/njs_vm.c                    |   77 +++++++++++++++++++++++
 njs/njs_vm.h                    |   12 +++
 njs/njscript.c                  |   38 +++++++++++
 njs/njscript.h                  |    8 ++
 njs/test/njs_interactive_test.c |  102 ++++++++++++++++++++++++++++++-
 9 files changed, 454 insertions(+), 6 deletions(-)

diffs (731 lines):

diff -r 746c49e6c195 -r 82dc332cb4c8 njs/njs.c
--- a/njs/njs.c	Thu Aug 24 16:16:37 2017 +0300
+++ b/njs/njs.c	Tue Aug 29 14:06:23 2017 +0300
@@ -59,6 +59,7 @@ static nxt_int_t njs_interactive_shell(n
 static nxt_int_t njs_process_file(njs_opts_t *opts, njs_vm_opt_t *vm_options);
 static nxt_int_t njs_process_script(njs_vm_t *vm, njs_opts_t *opts,
     const nxt_str_t *script, nxt_str_t *out);
+static void njs_print_backtrace(nxt_array_t *backtrace);
 static nxt_int_t njs_editline_init(njs_vm_t *vm);
 static char **njs_completion_handler(const char *text, int start, int end);
 static char *njs_completion_generator(const char *text, int state);
@@ -93,6 +94,7 @@ main(int argc, char **argv)
 
     vm_options.mcp = mcp;
     vm_options.accumulative = 1;
+    vm_options.backtrace = 1;
 
     if (opts.interactive) {
         ret = njs_interactive_shell(&opts, &vm_options);
@@ -150,9 +152,10 @@ njs_get_options(njs_opts_t *opts, int ar
 static nxt_int_t
 njs_interactive_shell(njs_opts_t *opts, njs_vm_opt_t *vm_options)
 {
-    njs_vm_t   *vm;
-    nxt_int_t  ret;
-    nxt_str_t  line, out;
+    njs_vm_t     *vm;
+    nxt_int_t    ret;
+    nxt_str_t    line, out;
+    nxt_array_t  *backtrace;
 
     vm = njs_vm_create(vm_options);
     if (vm == NULL) {
@@ -188,6 +191,11 @@ njs_interactive_shell(njs_opts_t *opts, 
 
         printf("%.*s\n", (int) out.length, out.start);
 
+        backtrace = njs_vm_backtrace(vm);
+        if (backtrace != NULL) {
+            njs_print_backtrace(backtrace);
+        }
+
         /* editline allocs a new buffer every time. */
         free(line.start);
     }
@@ -208,6 +216,7 @@ njs_process_file(njs_opts_t *opts, njs_v
     nxt_int_t    ret;
     nxt_str_t    out, script;
     struct stat  sb;
+    nxt_array_t  *backtrace;
 
     file = opts->file;
 
@@ -287,6 +296,11 @@ njs_process_file(njs_opts_t *opts, njs_v
 
     if (!opts->disassemble) {
         printf("%.*s\n", (int) out.length, out.start);
+
+        backtrace = njs_vm_backtrace(vm);
+        if (backtrace != NULL) {
+            njs_print_backtrace(backtrace);
+        }
     }
 
     return NXT_OK;
@@ -329,6 +343,26 @@ njs_process_script(njs_vm_t *vm, njs_opt
 }
 
 
+static void
+njs_print_backtrace(nxt_array_t *backtrace)
+{
+    nxt_uint_t             i;
+    njs_backtrace_entry_t  *be;
+
+    be = backtrace->start;
+
+    for (i = 0; i < backtrace->items; i++) {
+        if (be[i].line != 0) {
+            printf("at %.*s (:%d)\n", (int) be[i].name.length, be[i].name.start,
+                   be[i].line);
+
+        } else {
+            printf("at %.*s\n", (int) be[i].name.length, be[i].name.start);
+        }
+    }
+}
+
+
 static nxt_int_t
 njs_editline_init(njs_vm_t *vm)
 {
diff -r 746c49e6c195 -r 82dc332cb4c8 njs/njs_builtin.c
--- a/njs/njs_builtin.c	Thu Aug 24 16:16:37 2017 +0300
+++ b/njs/njs_builtin.c	Tue Aug 29 14:06:23 2017 +0300
@@ -514,3 +514,136 @@ njs_builtin_completions(njs_vm_t *vm, si
 
     return NXT_OK;
 }
+
+
+nxt_int_t
+njs_builtin_match_native_function(njs_vm_t *vm, njs_function_t *function,
+    nxt_str_t *name)
+{
+    char                    *buf;
+    size_t                  len;
+    nxt_str_t               string;
+    nxt_uint_t              i;
+    njs_object_t            *objects;
+    njs_function_t          *constructors;
+    njs_object_prop_t       *prop;
+    nxt_lvlhsh_each_t       lhe;
+    njs_object_prototype_t  *prototypes;
+
+    objects = vm->shared->objects;
+
+    for (i = NJS_OBJECT_THIS; i < NJS_OBJECT_MAX; i++) {
+        if (object_init[i] == NULL) {
+            continue;
+        }
+
+        nxt_lvlhsh_each_init(&lhe, &njs_object_hash_proto);
+
+        for ( ;; ) {
+            prop = nxt_lvlhsh_each(&objects[i].shared_hash, &lhe);
+
+            if (prop == NULL) {
+                break;
+            }
+
+            if (!njs_is_function(&prop->value)) {
+                continue;
+            }
+
+            if (function == prop->value.data.u.function) {
+                njs_string_get(&prop->name, &string);
+                len = object_init[i]->name.length + string.length
+                      + sizeof(".");
+
+                buf = nxt_mem_cache_zalloc(vm->mem_cache_pool, len);
+                if (buf == NULL) {
+                    return NXT_ERROR;
+                }
+
+                snprintf(buf, len, "%s.%s", object_init[i]->name.start,
+                         string.start);
+
+                name->length = len;
+                name->start = (u_char *) buf;
+
+                return NXT_OK;
+            }
+        }
+    }
+
+    prototypes = vm->shared->prototypes;
+
+    for (i = NJS_PROTOTYPE_OBJECT; i < NJS_PROTOTYPE_MAX; i++) {
+        nxt_lvlhsh_each_init(&lhe, &njs_object_hash_proto);
+
+        for ( ;; ) {
+            prop = nxt_lvlhsh_each(&prototypes[i].object.shared_hash, &lhe);
+
+            if (prop == NULL) {
+                break;
+            }
+
+            if (!njs_is_function(&prop->value)) {
+                continue;
+            }
+
+            if (function == prop->value.data.u.function) {
+                njs_string_get(&prop->name, &string);
+                len = prototype_init[i]->name.length + string.length
+                      + sizeof(".prototype.");
+
+                buf = nxt_mem_cache_zalloc(vm->mem_cache_pool, len);
+                if (buf == NULL) {
+                    return NXT_ERROR;
+                }
+
+                snprintf(buf, len, "%s.prototype.%s",
+                         prototype_init[i]->name.start, string.start);
+
+                name->length = len;
+                name->start = (u_char *) buf;
+
+                return NXT_OK;
+            }
+        }
+    }
+
+    constructors = vm->shared->constructors;
+
+    for (i = NJS_CONSTRUCTOR_OBJECT; i < NJS_CONSTRUCTOR_MAX; i++) {
+        nxt_lvlhsh_each_init(&lhe, &njs_object_hash_proto);
+
+        for ( ;; ) {
+            prop = nxt_lvlhsh_each(&constructors[i].object.shared_hash, &lhe);
+
+            if (prop == NULL) {
+                break;
+            }
+
+            if (!njs_is_function(&prop->value)) {
+                continue;
+            }
+
+            if (function == prop->value.data.u.function) {
+                njs_string_get(&prop->name, &string);
+                len = constructor_init[i]->name.length + string.length
+                      + sizeof(".");
+
+                buf = nxt_mem_cache_zalloc(vm->mem_cache_pool, len);
+                if (buf == NULL) {
+                    return NXT_ERROR;
+                }
+
+                snprintf(buf, len, "%s.%s", constructor_init[i]->name.start,
+                         string.start);
+
+                name->length = len;
+                name->start = (u_char *) buf;
+
+                return NXT_OK;
+            }
+        }
+    }
+
+    return NXT_DECLINED;
+}
diff -r 746c49e6c195 -r 82dc332cb4c8 njs/njs_generator.c
--- a/njs/njs_generator.c	Thu Aug 24 16:16:37 2017 +0300
+++ b/njs/njs_generator.c	Tue Aug 29 14:06:23 2017 +0300
@@ -123,6 +123,9 @@ static nxt_noinline nxt_int_t njs_genera
 static nxt_noinline nxt_int_t njs_generator_index_release(njs_vm_t *vm,
     njs_parser_t *parser, njs_index_t index);
 
+static nxt_int_t njs_generate_function_debug(njs_vm_t *vm, nxt_str_t *name,
+    njs_function_lambda_t *lambda, uint32_t line);
+
 
 static const nxt_str_t  no_label = { 0, NULL };
 
@@ -1609,6 +1612,13 @@ njs_generate_function(njs_vm_t *vm, njs_
         return ret;
     }
 
+    if (vm->debug != NULL) {
+        ret = njs_generate_function_debug(vm, NULL, lambda, node->token_line);
+        if (nxt_slow_path(ret != NXT_OK)) {
+            return ret;
+        }
+    }
+
     njs_generate_code(parser, njs_vmcode_function_t, function);
     function->code.operation = njs_vmcode_function;
     function->code.operands = NJS_VMCODE_1OPERAND;
@@ -1964,6 +1974,7 @@ static nxt_int_t
 njs_generate_function_declaration(njs_vm_t *vm, njs_parser_t *parser,
     njs_parser_node_t *node)
 {
+    nxt_int_t              ret;
     njs_variable_t         *var;
     njs_function_lambda_t  *lambda;
 
@@ -1974,7 +1985,17 @@ njs_generate_function_declaration(njs_vm
 
     lambda = var->value.data.u.function->u.lambda;
 
-    return njs_generate_function_scope(vm, lambda, node);
+    ret = njs_generate_function_scope(vm, lambda, node);
+    if (nxt_slow_path(ret != NXT_OK)) {
+        return ret;
+    }
+
+    if (vm->debug != NULL) {
+        ret = njs_generate_function_debug(vm, &var->name, lambda,
+                                          node->token_line);
+    }
+
+    return ret;
 }
 
 
@@ -2628,3 +2649,28 @@ njs_generator_index_release(njs_vm_t *vm
 
     return NXT_ERROR;
 }
+
+
+static nxt_int_t
+njs_generate_function_debug(njs_vm_t *vm, nxt_str_t *name,
+    njs_function_lambda_t *lambda, uint32_t line)
+{
+    njs_function_debug_t  *debug;
+
+    debug = nxt_array_add(vm->debug, &njs_array_mem_proto, vm->mem_cache_pool);
+    if (nxt_slow_path(debug == NULL)) {
+        return NXT_ERROR;
+    }
+
+    if (name != NULL) {
+        debug->name = *name;
+
+    } else {
+        debug->name = no_label;
+    }
+
+    debug->lambda = lambda;
+    debug->line = line;
+
+    return NXT_OK;
+}
diff -r 746c49e6c195 -r 82dc332cb4c8 njs/njs_parser.c
--- a/njs/njs_parser.c	Thu Aug 24 16:16:37 2017 +0300
+++ b/njs/njs_parser.c	Tue Aug 29 14:06:23 2017 +0300
@@ -465,6 +465,7 @@ njs_parser_function_declaration(njs_vm_t
     }
 
     node->token = NJS_TOKEN_FUNCTION;
+    node->token_line = parser->lexer->token_line;
 
     token = njs_parser_token(parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
@@ -532,6 +533,7 @@ njs_parser_function_expression(njs_vm_t 
     }
 
     node->token = NJS_TOKEN_FUNCTION_EXPRESSION;
+    node->token_line = parser->lexer->token_line;
     node->scope = parser->scope;
     parser->node = node;
     parser->code_size += sizeof(njs_vmcode_function_t);
diff -r 746c49e6c195 -r 82dc332cb4c8 njs/njs_vm.c
--- a/njs/njs_vm.c	Thu Aug 24 16:16:37 2017 +0300
+++ b/njs/njs_vm.c	Tue Aug 29 14:06:23 2017 +0300
@@ -113,6 +113,8 @@ static njs_ret_t njs_object_value_to_str
 static njs_ret_t njs_vmcode_value_to_string(njs_vm_t *vm, njs_value_t *invld1,
     njs_value_t *invld2);
 
+static njs_ret_t njs_vm_add_backtrace_entry(njs_vm_t *vm, njs_frame_t *frame);
+
 void njs_debug(njs_index_t index, njs_value_t *value);
 
 
@@ -264,9 +266,20 @@ start:
 
             if (catch != NULL) {
                 vm->current = catch;
+
+                if (vm->debug != NULL) {
+                    nxt_array_reset(vm->backtrace);
+                }
+
                 goto start;
             }
 
+            if (vm->debug != NULL
+                && njs_vm_add_backtrace_entry(vm, frame) != NXT_OK)
+            {
+                return NXT_ERROR;
+            }
+
             previous = frame->native.previous;
             if (previous == NULL) {
                 return NXT_ERROR;
@@ -3433,6 +3446,70 @@ njs_vm_throw_exception(njs_vm_t *vm, con
 }
 
 
+static njs_ret_t
+njs_vm_add_backtrace_entry(njs_vm_t *vm, njs_frame_t *frame)
+{
+    nxt_int_t              ret;
+    nxt_uint_t             i;
+    njs_function_t         *function;
+    njs_native_frame_t     *native_frame;
+    njs_function_debug_t   *debug_entry;
+    njs_function_lambda_t  *lambda;
+    njs_backtrace_entry_t  *be;
+
+    static const nxt_str_t  entry_main =        nxt_string("main");
+    static const nxt_str_t  entry_native =      nxt_string("native");
+    static const nxt_str_t  entry_unknown =     nxt_string("unknown");
+    static const nxt_str_t  entry_anonymous =   nxt_string("anonymous");
+
+    native_frame = &frame->native;
+    function = native_frame->function;
+
+    be = nxt_array_add(vm->backtrace, &njs_array_mem_proto, vm->mem_cache_pool);
+    if (nxt_slow_path(be == NULL)) {
+        return NXT_ERROR;
+    }
+
+    be->line = 0;
+
+    if (function == NULL) {
+        be->name = entry_main;
+        return NXT_OK;
+    }
+
+    if (function->native) {
+        ret = njs_builtin_match_native_function(vm, function, &be->name);
+        if (ret != NXT_OK) {
+            be->name = entry_native;
+        }
+
+        return NXT_OK;
+    }
+
+    lambda = function->u.lambda;
+    debug_entry = vm->debug->start;
+
+    for (i = 0; i < vm->debug->items; i++) {
+        if (lambda == debug_entry[i].lambda) {
+            if (debug_entry[i].name.length != 0) {
+                be->name = debug_entry[i].name;
+
+            } else {
+                be->name = entry_anonymous;
+            }
+
+            be->line = debug_entry[i].line;
+
+            return NXT_OK;
+        }
+    }
+
+    be->name = entry_unknown;
+
+    return NXT_OK;
+}
+
+
 void
 njs_debug(njs_index_t index, njs_value_t *value)
 {
diff -r 746c49e6c195 -r 82dc332cb4c8 njs/njs_vm.h
--- a/njs/njs_vm.h	Thu Aug 24 16:16:37 2017 +0300
+++ b/njs/njs_vm.h	Tue Aug 29 14:06:23 2017 +0300
@@ -853,6 +853,13 @@ typedef struct {
 } njs_vm_trap_t;
 
 
+typedef struct {
+    uint32_t                  line;
+    nxt_str_t                 name;
+    njs_function_lambda_t     *lambda;
+} njs_function_debug_t;
+
+
 struct njs_vm_s {
     /* njs_vm_t must be aligned to njs_value_t due to scratch value. */
     njs_value_t              retval;
@@ -903,6 +910,9 @@ struct njs_vm_s {
     nxt_trace_t              trace;
     nxt_random_t             random;
 
+    nxt_array_t              *debug;
+    nxt_array_t              *backtrace;
+
     uint8_t                  trailer;  /* 1 bit */
     uint8_t                  accumulative; /* 1 bit */
 };
@@ -1080,6 +1090,8 @@ void njs_vm_throw_exception(njs_vm_t *vm
 
 nxt_int_t njs_builtin_objects_create(njs_vm_t *vm);
 nxt_int_t njs_builtin_objects_clone(njs_vm_t *vm);
+nxt_int_t njs_builtin_match_native_function(njs_vm_t *vm,
+    njs_function_t *function, nxt_str_t *name);
 
 
 void *njs_lvlhsh_alloc(void *data, size_t size, nxt_uint_t nalloc);
diff -r 746c49e6c195 -r 82dc332cb4c8 njs/njscript.c
--- a/njs/njscript.c	Thu Aug 24 16:16:37 2017 +0300
+++ b/njs/njscript.c	Tue Aug 29 14:06:23 2017 +0300
@@ -106,6 +106,7 @@ njs_vm_create(njs_vm_opt_t *options)
 {
     njs_vm_t              *vm;
     nxt_int_t             ret;
+    nxt_array_t           *debug;
     nxt_mem_cache_pool_t  *mcp;
     njs_regexp_pattern_t  *pattern;
 
@@ -176,6 +177,17 @@ njs_vm_create(njs_vm_opt_t *options)
 
         vm->trailer = options->trailer;
 
+        if (options->backtrace) {
+            debug = nxt_array_create(4, sizeof(njs_function_debug_t),
+                                     &njs_array_mem_proto,
+                                     vm->mem_cache_pool);
+            if (nxt_slow_path(debug == NULL)) {
+                return NULL;
+            }
+
+            vm->debug = debug;
+        }
+
         vm->accumulative = options->accumulative;
         if (vm->accumulative) {
             ret = njs_vm_init(vm);
@@ -248,6 +260,10 @@ njs_vm_compile(njs_vm_t *vm, u_char **st
      */
     vm->code = NULL;
 
+    if (vm->backtrace != NULL) {
+        nxt_array_reset(vm->backtrace);
+    }
+
     ret = njs_generate_scope(vm, parser, node);
     if (nxt_slow_path(ret != NXT_OK)) {
         goto fail;
@@ -334,6 +350,7 @@ njs_vm_init(njs_vm_t *vm)
     u_char       *values;
     nxt_int_t    ret;
     njs_frame_t  *frame;
+    nxt_array_t  *backtrace;
 
     scope_size = vm->scope_size + NJS_INDEX_GLOBAL_OFFSET;
 
@@ -371,6 +388,16 @@ njs_vm_init(njs_vm_t *vm)
         return NXT_ERROR;
     }
 
+    if (vm->debug != NULL) {
+        backtrace = nxt_array_create(4, sizeof(njs_backtrace_entry_t),
+                                     &njs_array_mem_proto, vm->mem_cache_pool);
+        if (nxt_slow_path(backtrace == NULL)) {
+            return NXT_ERROR;
+        }
+
+        vm->backtrace = backtrace;
+    }
+
     vm->retval = njs_value_void;
 
     vm->trace.level = NXT_LEVEL_TRACE;
@@ -493,3 +520,14 @@ njs_vm_exception(njs_vm_t *vm, nxt_str_t
 {
     return njs_value_to_ext_string(vm, retval, vm->exception);
 }
+
+
+nxt_array_t *
+njs_vm_backtrace(njs_vm_t *vm)
+{
+    if (!nxt_array_is_empty(vm->backtrace)) {
+        return vm->backtrace;
+    }
+
+    return NULL;
+}
diff -r 746c49e6c195 -r 82dc332cb4c8 njs/njscript.h
--- a/njs/njscript.h	Thu Aug 24 16:16:37 2017 +0300
+++ b/njs/njscript.h	Tue Aug 29 14:06:23 2017 +0300
@@ -72,9 +72,16 @@ typedef struct {
 
     uint8_t                         trailer;         /* 1 bit */
     uint8_t                         accumulative;    /* 1 bit */
+    uint8_t                         backtrace;       /* 1 bit */
 } njs_vm_opt_t;
 
 
+typedef struct {
+    nxt_str_t                       name;
+    uint32_t                        line;
+} njs_backtrace_entry_t;
+
+
 #define NJS_OK                      NXT_OK
 #define NJS_ERROR                   NXT_ERROR
 #define NJS_AGAIN                   NXT_AGAIN
@@ -103,6 +110,7 @@ NXT_EXPORT njs_ret_t njs_vm_return_strin
     size_t size);
 NXT_EXPORT nxt_int_t njs_vm_retval(njs_vm_t *vm, nxt_str_t *retval);
 NXT_EXPORT nxt_int_t njs_vm_exception(njs_vm_t *vm, nxt_str_t *retval);
+NXT_EXPORT nxt_array_t *njs_vm_backtrace(njs_vm_t *vm);
 
 NXT_EXPORT void njs_disassembler(njs_vm_t *vm);
 
diff -r 746c49e6c195 -r 82dc332cb4c8 njs/test/njs_interactive_test.c
--- a/njs/test/njs_interactive_test.c	Thu Aug 24 16:16:37 2017 +0300
+++ b/njs/test/njs_interactive_test.c	Tue Aug 29 14:06:23 2017 +0300
@@ -110,17 +110,80 @@ static njs_interactive_test_t  njs_test[
       nxt_string("4") },
 
     { nxt_string("function ff(o) {return o.a.a}" ENTER
+                 "function f(o) {try {return ff(o)} "
+                                 "finally {return 1}}" ENTER
+                 "f({})" ENTER),
+      nxt_string("1") },
+
+    /* Backtraces */
+
+    { nxt_string("function ff(o) {return o.a.a}" ENTER
                  "function f(o) {return ff(o)}" ENTER
                  "f({})" ENTER),
-      nxt_string("TypeError") },
+      nxt_string("TypeError\n"
+                 "at ff (:1)\n"
+                 "at f (:1)\n"
+                 "at main\n") },
+
+    { nxt_string("function ff(o) {return o.a.a}" ENTER
+                 "function f(o) {try {return ff(o)} "
+                                 "finally {return o.a.a}}" ENTER
+                 "f({})" ENTER),
+      nxt_string("TypeError\n"
+                 "at f (:1)\n"
+                 "at main\n") },
 
     { nxt_string("function f(ff, o) {return ff(o)}" ENTER
                  "f(function (o) {return o.a.a}, {})" ENTER),
-      nxt_string("TypeError") },
+      nxt_string("TypeError\n"
+                 "at anonymous (:1)\n"
+                 "at f (:1)\n"
+                 "at main\n") },
+
+    { nxt_string("'str'.replace(/t/g,"
+                 "              function(m) {return m.a.a})" ENTER),
+      nxt_string("TypeError\n"
+                 "at anonymous (:1)\n"
+                 "at String.prototype.replace\n"
+                 "at main\n") },
+
+    { nxt_string("function f(o) {return Object.keys(o)}" ENTER
+                 "f()" ENTER),
+      nxt_string("TypeError\n"
+                 "at Object.keys\n"
+                 "at f (:1)\n"
+                 "at main\n") },
+
+    { nxt_string("String.fromCharCode(3.14)" ENTER),
+      nxt_string("RangeError\n"
+                 "at String.fromCharCode\n"
+                 "at main\n") },
+
+    { nxt_string("Math.log({}.a.a)" ENTER),
+      nxt_string("TypeError\n"
+                 "at Math.log\n"
+                 "at main\n") },
+
+    { nxt_string("function f(o) {function f_in(o) {return o.a.a};"
+                 "               return f_in(o)}; f({})" ENTER),
+      nxt_string("TypeError\n"
+                 "at f_in (:1)\n"
+                 "at f (:1)\n"
+                 "at main\n") },
+
+    { nxt_string("function f(o) {var ff = function (o) {return o.a.a};"
+                 "               return ff(o)}; f({})" ENTER),
+      nxt_string("TypeError\n"
+                 "at anonymous (:1)\n"
+                 "at f (:1)\n"
+                 "at main\n") },
 
 };
 
 
+static void njs_report_backtrace(nxt_array_t *backtrace, nxt_str_t *s);
+
+
 static nxt_int_t
 njs_interactive_test(void)
 {
@@ -130,6 +193,7 @@ njs_interactive_test(void)
     nxt_str_t               s;
     nxt_uint_t              i;
     nxt_bool_t              success;
+    nxt_array_t             *backtrace;
     njs_vm_opt_t            options;
     nxt_mem_cache_pool_t    *mcp;
     njs_interactive_test_t  *test;
@@ -153,6 +217,7 @@ njs_interactive_test(void)
 
         options.mcp = mcp;
         options.accumulative = 1;
+        options.backtrace = 1;
 
         vm = njs_vm_create(&options);
         if (vm == NULL) {
@@ -184,6 +249,11 @@ njs_interactive_test(void)
 
         } else {
             njs_vm_exception(vm, &s);
+
+            backtrace = njs_vm_backtrace(vm);
+            if (backtrace != NULL) {
+                njs_report_backtrace(backtrace, &s);
+            }
         }
 
         success = nxt_strstr_eq(&test->ret, &s);
@@ -211,6 +281,34 @@ fail:
 }
 
 
+static void
+njs_report_backtrace(nxt_array_t *backtrace, nxt_str_t *s)
+{
+    char                   *p;
+    nxt_uint_t             i;
+    njs_backtrace_entry_t  *be;
+
+    static char            buf[4096];
+
+    p = buf + sprintf(buf, "%.*s\n", (int) s->length, s->start);
+
+    be = backtrace->start;
+    for (i = 0; i < backtrace->items; i++) {
+        if (be[i].line != 0) {
+            p += sprintf(p, "at %.*s (:%d)\n", (int) be[i].name.length,
+                         be[i].name.start, be[i].line);
+
+        } else {
+            p += sprintf(p, "at %.*s\n", (int) be[i].name.length,
+                         be[i].name.start);
+        }
+    }
+
+    s->length = strlen(buf);
+    s->start = (u_char *) buf;
+}
+
+
 int nxt_cdecl
 main(int argc, char **argv)
 {

From piotrsikora at google.com  Wed Aug 30 21:55:34 2017
From: piotrsikora at google.com (Piotr Sikora)
Date: Wed, 30 Aug 2017 14:55:34 -0700
Subject: [PATCH] HTTP/2: signal 0-byte HPACK's dynamic table size
Message-ID: <7e1e91f9ca063563cb29.1504130134@piotrsikora.sfo.corp.google.com>

# HG changeset patch
# User Piotr Sikora <piotrsikora at google.com>
# Date 1504129931 25200
#      Wed Aug 30 14:52:11 2017 -0700
# Node ID 7e1e91f9ca063563cb293136e7b1cede36e63dc6
# Parent  c7d4017c8876af6d8570e400320537d7d39e9578
HTTP/2: signal 0-byte HPACK's dynamic table size.

This change lets NGINX talk to clients with SETTINGS_HEADER_TABLE_SIZE
smaller than the default 4KB. Previously, NGINX would ACK the SETTINGS
frame with a small dynamic table size, but it would never send dynamic
table size update, leading to a connection-level COMPRESSION_ERROR.

Also, it allows clients to release 4KB of memory per connection, since
NGINX doesn't use HPACK's dynamic table when encoding headers, however
clients had to maintain it, since NGINX never signaled that it doesn't
use it.

Signed-off-by: Piotr Sikora <piotrsikora at google.com>

diff -r c7d4017c8876 -r 7e1e91f9ca06 src/http/v2/ngx_http_v2.c
--- a/src/http/v2/ngx_http_v2.c
+++ b/src/http/v2/ngx_http_v2.c
@@ -245,6 +245,8 @@ ngx_http_v2_init(ngx_event_t *rev)
 
     h2c->frame_size = NGX_HTTP_V2_DEFAULT_FRAME_SIZE;
 
+    h2c->table_update = 1;
+
     h2scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v2_module);
 
     h2c->pool = ngx_create_pool(h2scf->pool_size, h2c->connection->log);
diff -r c7d4017c8876 -r 7e1e91f9ca06 src/http/v2/ngx_http_v2.h
--- a/src/http/v2/ngx_http_v2.h
+++ b/src/http/v2/ngx_http_v2.h
@@ -144,6 +144,7 @@ struct ngx_http_v2_connection_s {
 
     unsigned                         closed_nodes:8;
     unsigned                         settings_ack:1;
+    unsigned                         table_update:1;
     unsigned                         blocked:1;
     unsigned                         goaway:1;
 };
diff -r c7d4017c8876 -r 7e1e91f9ca06 src/http/v2/ngx_http_v2_filter_module.c
--- a/src/http/v2/ngx_http_v2_filter_module.c
+++ b/src/http/v2/ngx_http_v2_filter_module.c
@@ -141,6 +141,7 @@ ngx_http_v2_header_filter(ngx_http_reque
     ngx_http_v2_out_frame_t   *frame;
     ngx_http_core_loc_conf_t  *clcf;
     ngx_http_core_srv_conf_t  *cscf;
+    ngx_http_v2_connection_t  *h2c;
     u_char                     addr[NGX_SOCKADDR_STRLEN];
 
     static const u_char nginx[5] = "\x84\xaa\x63\x55\xe7";
@@ -235,7 +236,11 @@ ngx_http_v2_header_filter(ngx_http_reque
         }
     }
 
-    len = status ? 1 : 1 + ngx_http_v2_literal_size("418");
+    h2c = r->stream->connection;
+
+    len = h2c->table_update ? 1 : 0;
+
+    len += status ? 1 : 1 + ngx_http_v2_literal_size("418");
 
     clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
 
@@ -423,6 +428,13 @@ ngx_http_v2_header_filter(ngx_http_reque
 
     start = pos;
 
+    if (h2c->table_update) {
+        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0,
+                       "http2 sending dynamic table size update: 0");
+        *pos++ = 32;
+        h2c->table_update = 0;
+    }
+
     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0,
                    "http2 output header: \":status: %03ui\"",
                    r->headers_out.status);

From piotrsikora at google.com  Wed Aug 30 21:55:21 2017
From: piotrsikora at google.com (Piotr Sikora)
Date: Wed, 30 Aug 2017 14:55:21 -0700
Subject: [PATCH] HTTP/2: don't limit number of requests per HTTP/2 connection
Message-ID: <49b677bf2ae7ab924997.1504130121@piotrsikora.sfo.corp.google.com>

# HG changeset patch
# User Piotr Sikora <piotrsikora at google.com>
# Date 1504129797 25200
#      Wed Aug 30 14:49:57 2017 -0700
# Node ID 49b677bf2ae7ab92499766e8184ddcbf7a4233f9
# Parent  c7d4017c8876af6d8570e400320537d7d39e9578
HTTP/2: don't limit number of requests per HTTP/2 connection.

Previous default limit (1000 requests) and lack of graceful shutdown
could result in loss of requests, when clients were unable to retry.

Signed-off-by: Piotr Sikora <piotrsikora at google.com>

diff -r c7d4017c8876 -r 49b677bf2ae7 src/http/v2/ngx_http_v2_module.c
--- a/src/http/v2/ngx_http_v2_module.c
+++ b/src/http/v2/ngx_http_v2_module.c
@@ -11,6 +11,9 @@
 #include <ngx_http_v2_module.h>
 
 
+#define NGX_HTTP_V2_MAX_STREAMS  (1U << 30)
+
+
 static ngx_int_t ngx_http_v2_add_variables(ngx_conf_t *cf);
 
 static ngx_int_t ngx_http_v2_variable(ngx_http_request_t *r,
@@ -355,7 +358,8 @@ ngx_http_v2_merge_srv_conf(ngx_conf_t *c
 
     ngx_conf_merge_uint_value(conf->concurrent_streams,
                               prev->concurrent_streams, 128);
-    ngx_conf_merge_uint_value(conf->max_requests, prev->max_requests, 1000);
+    ngx_conf_merge_uint_value(conf->max_requests, prev->max_requests,
+                              NGX_HTTP_V2_MAX_STREAMS);
 
     ngx_conf_merge_size_value(conf->max_field_size, prev->max_field_size,
                               4096);

From vbart at nginx.com  Wed Aug 30 22:54:31 2017
From: vbart at nginx.com (Valentin V. Bartenev)
Date: Thu, 31 Aug 2017 01:54:31 +0300
Subject: [PATCH] HTTP/2: don't limit number of requests per HTTP/2
 connection
In-Reply-To: <49b677bf2ae7ab924997.1504130121@piotrsikora.sfo.corp.google.com>
References: <49b677bf2ae7ab924997.1504130121@piotrsikora.sfo.corp.google.com>
Message-ID: <5946921.3WlCo78QHa@vbart-laptop>

On ???????, 31 ??????? 2017 ?. 0:55:21 MSK Piotr Sikora via nginx-devel wrote:
> # HG changeset patch
> # User Piotr Sikora <piotrsikora at google.com>
> # Date 1504129797 25200
> #      Wed Aug 30 14:49:57 2017 -0700
> # Node ID 49b677bf2ae7ab92499766e8184ddcbf7a4233f9
> # Parent  c7d4017c8876af6d8570e400320537d7d39e9578
> HTTP/2: don't limit number of requests per HTTP/2 connection.
> 
> Previous default limit (1000 requests) and lack of graceful shutdown
> could result in loss of requests, when clients were unable to retry.
> 
> Signed-off-by: Piotr Sikora <piotrsikora at google.com>
>
[..]

This opens a vector for dos attack.  There are some configurations
when memory can be allocated from connection pool for each request.
Removing a reasonable enough limit for requests per connection
potentially allow an attacker to grow this pool until a worker
process will be killed due to OOM.

The problem should be solved by introducing "lingering close",
similar to HTTP/1.x.

  wbr, Valentin V. Bartenev


From piotrsikora at google.com  Wed Aug 30 23:14:03 2017
From: piotrsikora at google.com (Piotr Sikora)
Date: Wed, 30 Aug 2017 16:14:03 -0700
Subject: [PATCH] HTTP/2: don't limit number of requests per HTTP/2
 connection
In-Reply-To: <5946921.3WlCo78QHa@vbart-laptop>
References: <49b677bf2ae7ab924997.1504130121@piotrsikora.sfo.corp.google.com>
 <5946921.3WlCo78QHa@vbart-laptop>
Message-ID: <CAF-CG+Kt+UrRgbO8ykhHeWQ4MiYpwusAbmue_2jH4sh=bF_BLA@mail.gmail.com>

Hey Valentin,

> This opens a vector for dos attack.  There are some configurations
> when memory can be allocated from connection pool for each request.
> Removing a reasonable enough limit for requests per connection
> potentially allow an attacker to grow this pool until a worker
> process will be killed due to OOM.
>
> The problem should be solved by introducing "lingering close",
> similar to HTTP/1.x.

Yes, the proper solution is graceful shutdown via 2-stage GOAWAY,
as defined in RFC7540, Section 6.8, but I don't have capacity to
work on it now, and above patch is IMHO better than lost requests.

Best regards,
Piotr Sikora

From mdounin at mdounin.ru  Thu Aug 31 14:42:31 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 31 Aug 2017 17:42:31 +0300
Subject: [PATCH] [PATCH 1 of 4] Core: add function to decode hexadecimal
 strings
In-Reply-To: <17c038b56051f922ec44.1503541179@OLA-6J5NNS1.ad.garmin.com>
References: <010418>
 <17c038b56051f922ec44.1503541179@OLA-6J5NNS1.ad.garmin.com>
Message-ID: <20170831144231.GD93611@mdounin.ru>

Hello!

On Wed, Aug 23, 2017 at 09:19:39PM -0500, Nate Karstens wrote:

> # HG changeset patch
> # User Nate Karstens <nate.karstens at garmin.com>
> # Date 1503540018 18000
> #      Wed Aug 23 21:00:18 2017 -0500
> # Node ID 17c038b56051f922ec440d40e23e8d1b23d835df
> # Parent  c7d4017c8876af6d8570e400320537d7d39e9578
> [PATCH 1 of 4] Core: add function to decode hexadecimal strings.

There is no need to include "[PATCH 1 of 4] " into the patch 
summary line.

> 
> Adds functionality to convert a hexadecimal string into binary data.
> This will be used to decode PSKs stored in hexadecimal representation.
> 
> Signed-off-by: Nate Karstens <nate.karstens at garmin.com>
> 
> diff -r c7d4017c8876 -r 17c038b56051 src/core/ngx_string.c
> --- a/src/core/ngx_string.c     Tue Aug 22 21:22:59 2017 +0300
> +++ b/src/core/ngx_string.c     Wed Aug 23 21:00:18 2017 -0500
> @@ -1118,6 +1118,57 @@ ngx_hex_dump(u_char *dst, u_char *src, s
>  }
> 
> 
> +ngx_int_t
> +ngx_hex_decode(u_char *dst, u_char *src, size_t len)
> +{
> +    u_char     ch, decoded;

Style: there should be two spaces between the (longest) type and 
variables.

> +
> +    if (len & 1) {
> +        return NGX_ERROR;
> +    }
> +
> +    while (len > 0) {
> +        ch = *src++;
> +        len--;

It might worth rewriting this to be more in line with 
ngx_hex_dump() as

    while (len--) {
        ch = *src++;

> +
> +        if (len & 1) {
> +
> +            if (ch >= '0' && ch <= '9') {
> +                decoded = ch - '0';
> +                continue;
> +            }
> +
> +            ch |= 0x20;
> +
> +            if (ch >= 'a' && ch <= 'f') {
> +                decoded = ch - 'a' + 10;
> +                continue;
> +            }
> +
> +            return NGX_ERROR;
> +
> +        } else {
> +
> +            if (ch >= '0' && ch <= '9') {
> +                *dst++ = (u_char) ((decoded << 4) + ch - '0');

Here compilation with at least Clang fails with the following error:

src/core/ngx_string.c:1153:37: error: variable 'decoded' may be uninitialized
      when used here [-Werror,-Wconditional-uninitialized]
                *dst++ = (u_char) ((decoded << 4) + ch - '0');
                                    ^~~~~~~

This looks like false positive, though needs fixing anyway.  For 
me, most trivial solution would be to unroll the loop instead of 
relying on the non-trivial (len & 1) check.

[...]

-- 
Maxim Dounin
http://nginx.org/

From mdounin at mdounin.ru  Thu Aug 31 14:43:45 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 31 Aug 2017 17:43:45 +0300
Subject: [PATCH] [PATCH 2 of 4] SSL: add support for PSK cipher suites
In-Reply-To: <97953fe374455a049732.1503541250@OLA-6J5NNS1.ad.garmin.com>
References: <010419>
 <97953fe374455a049732.1503541250@OLA-6J5NNS1.ad.garmin.com>
Message-ID: <20170831144345.GE93611@mdounin.ru>

Hello!

On Wed, Aug 23, 2017 at 09:20:50PM -0500, Nate Karstens wrote:

> # HG changeset patch
> # User Nate Karstens <nate.karstens at garmin.com>
> # Date 1503540059 18000
> #      Wed Aug 23 21:00:59 2017 -0500
> # Node ID 97953fe374455a04973268c4b2fbadd7ced91ffe
> # Parent  17c038b56051f922ec440d40e23e8d1b23d835df
> [PATCH 2 of 4] SSL: add support for PSK cipher suites.

There is no need to include "[PATCH 2 of 4] " into patch summary 
line.

[...]

> @@ -1163,6 +1177,211 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_s
> 
> 
>  ngx_int_t
> +ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file)
> +{
> +#ifdef PSK_MAX_IDENTITY_LEN
> +
> +    ngx_fd_t            fd;
> +    ngx_int_t           err;
> +    ngx_file_info_t     fi;

Style: there should be two spaces between the longest type and a 
variable.  I've already explained details in one of the previous 
reviews here:

http://mailman.nginx.org/pipermail/nginx-devel/2017-June/010247.html 

> +
> +    err = NGX_OK;
> +
> +    if (ngx_conf_full_name(cf->cycle, file, 1) != NGX_OK) {
> +        return NGX_ERROR;
> +    }
> +
> +    fd = ngx_open_file(file->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
> +    if (fd == NGX_INVALID_FILE) {
> +        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, ngx_errno,
> +                      ngx_open_file_n " \"%V\" failed", file);
> +        return NGX_ERROR;
> +    }
> +
> +    if (ngx_fd_info(fd, &fi) == NGX_FILE_ERROR) {
> +        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, ngx_errno,
> +                      ngx_fd_info_n " \"%V\" failed", file);
> +        err = NGX_ERROR;
> +        goto failed;
> +    }
> +
> +    if (ngx_file_size(&fi) == 0) {
> +        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
> +                      "PSK file \"%V\" is empty", file);
> +        err = NGX_ERROR;
> +        goto failed;
> +    }

What's the point in opening the file here?  As long the file is 
being read at run-time, it doesn't seem to make sense.  The file 
can be legitimately created later.

Checking ngx_file_size() looks completely wrong as well.  It will 
prevent configurations with no currently configured PSK keys from 
running if the file size is 0 - but won't do this as long as there 
is a comment and/or some garbage in the file.

> +
> +failed:
> +
> +    if (ngx_close_file(fd) == NGX_FILE_ERROR) {
> +        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, ngx_errno,
> +                      ngx_close_file_n " %V failed", file);
> +        err = NGX_ERROR;
> +    }
> +
> +    if (err != NGX_OK) {
> +        return err;
> +    }
> +
> +    if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_psk_index, file) == 0) {
> +        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
> +                      "SSL_CTX_set_ex_data() failed");
> +        return NGX_ERROR;
> +    }
> +
> +    SSL_CTX_set_psk_server_callback(ssl->ctx, ngx_ssl_psk_callback);
> +
> +#endif
> +
> +    return NGX_OK;
> +}
> +
> +
> +#ifdef PSK_MAX_IDENTITY_LEN
> +
> +static unsigned int
> +ngx_ssl_psk_callback(ngx_ssl_conn_t *ssl_conn, const char *identity,
> +    unsigned char *psk, unsigned int max_psk_len)
> +{
> +    u_char             *p, *last, *end, *colon;
> +    size_t              len;
> +    ssize_t             n;
> +    SSL_CTX            *ssl_ctx;
> +    ngx_fd_t            fd;
> +    ngx_str_t          *file;
> +    unsigned int        psk_len;
> +    ngx_connection_t   *c;
> +    u_char              buf[NGX_SSL_PSK_BUFFER_SIZE];

Style: there should be two spaces between "ngx_connection_t" and 
"*c".

> +
> +    psk_len = 0;

This looks to early for the psk_len initialization.  Rather, I 
would move the initialization to somewhere near "len = 0; last = 
buf" below where it will look more logical.

> +    c = ngx_ssl_get_connection(ssl_conn);
> +
> +    ssl_ctx = SSL_get_SSL_CTX(ssl_conn);
> +    file = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_psk_index);
> +
> +    fd = ngx_open_file(file->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
> +    if (fd == NGX_INVALID_FILE) {
> +        ngx_ssl_error(NGX_LOG_ERR, c->log, ngx_errno,
> +                      ngx_open_file_n " \"%V\" failed", file);
> +        return 0;
> +    }
> +
> +    len = 0;
> +    last = buf;
> +
> +    do {
> +        n = ngx_read_fd(fd, last, NGX_SSL_PSK_BUFFER_SIZE - len);
> +
> +        if (n == -1) {
> +            ngx_ssl_error(NGX_LOG_ERR, c->log, ngx_errno,
> +                          ngx_read_fd_n " \"%V\" failed", file);
> +            psk_len = 0;

There is no need to set psk_len to 0 here, as it is already 
initialized to 0 and never set to anything else except before 
"goto cleanup".

> +            goto cleanup;
> +        }
> +
> +        end = last + n;
> +
> +        if (len && n == 0) {
> +            *end++ = LF;
> +        }
> +
> +        for (p = buf; /* void */; p = last) {
> +            last = ngx_strlchr(last, end, LF);
> +
> +            if (last == NULL) {
> +                break;
> +            }
> +
> +            len = last++ - p;
> +
> +            if (len && p[len - 1] == CR) {
> +                len--;
> +            }
> +
> +            if (len == 0) {
> +                continue;
> +            }
> +
> +            colon = ngx_strlchr(p, p + len, ':');
> +
> +            if (colon == NULL) {
> +                continue;
> +            }
> +
> +            *colon = '\0';
> +
> +            if (ngx_strcmp(p, identity) != 0) {
> +                continue;
> +            }
> +
> +            len -= colon + 1 - p;
> +            p = colon + 1;
> +
> +            if (ngx_strncmp(p, "{HEX}", sizeof("{HEX}") - 1) == 0) {
> +
> +                p += sizeof("{HEX}") - 1;
> +                len -= sizeof("{HEX}") - 1;
> +
> +                if (len / 2 > max_psk_len) {
> +                    goto cleanup;
> +                }
> +
> +                if (ngx_hex_decode(psk, p, len) != NGX_OK) {
> +                    ngx_memzero(psk, len / 2);
> +                    goto cleanup;
> +                }
> +
> +                psk_len = len / 2;
> +
> +                goto cleanup;
> +
> +            } else if (ngx_strncmp(p, "{PLAIN}", sizeof("{PLAIN}") - 1) == 0) {
> +                p += sizeof("{PLAIN}") - 1;
> +                len -= sizeof("{PLAIN}") - 1;
> +            }
> +
> +            if (len > max_psk_len) {
> +                goto cleanup;
> +            }
> +
> +            ngx_memcpy(psk, p, len);
> +            psk_len = len;
> +
> +            goto cleanup;
> +        }
> +
> +        len = end - p;
> +
> +        if (len == NGX_SSL_PSK_BUFFER_SIZE) {
> +            ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
> +                          "too long line in \"%V\"", file);
> +            psk_len = 0;

There is no need to set psk_len here, see above.

> +            goto cleanup;
> +        }
> +
> +        ngx_memmove(buf, p, len);
> +        last = buf + len;
> +
> +    } while (n != 0);
> +
> +cleanup:
> +
> +    if (ngx_close_file(fd) == NGX_FILE_ERROR) {
> +        ngx_ssl_error(NGX_LOG_ALERT, c->log, ngx_errno,
> +                      ngx_close_file_n " %V failed", file);
> +        psk_len = 0;

I don't think we should reset psk_len here, this error is not 
related to the PSK key we already either have at this point, or 
psk_len is already set to 0.

> +    }
> +
> +    ngx_memzero(buf, NGX_SSL_PSK_BUFFER_SIZE);
> +
> +    return psk_len;
> +}
> +
> +#endif
> +
> +
> +ngx_int_t
>  ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags)
>  {
>      ngx_ssl_connection_t  *sc;

In the previous review I've pointed out that the patch needs to 
adjust severity levels in ngx_ssl_connection_error():

http://mailman.nginx.org/pipermail/nginx-devel/2017-August/010419.html

Are there any specific reasons why you've ignored this comment?  
Just in case, here is the relevant change:

@@ -2249,6 +2249,9 @@ ngx_ssl_connection_error(ngx_connection_
             || n == SSL_R_NO_COMPRESSION_SPECIFIED                   /*  187 */
             || n == SSL_R_NO_SHARED_CIPHER                           /*  193 */
             || n == SSL_R_RECORD_LENGTH_MISMATCH                     /*  213 */
+#ifdef SSL_R_PSK_IDENTITY_NOT_FOUND
+            || n == SSL_R_PSK_IDENTITY_NOT_FOUND                     /*  223 */
+#endif
 #ifdef SSL_R_PARSE_TLSEXT
             || n == SSL_R_PARSE_TLSEXT                               /*  227 */
 #endif


> diff -r 17c038b56051 -r 97953fe37445 src/event/ngx_event_openssl.h
> --- a/src/event/ngx_event_openssl.h     Wed Aug 23 21:00:18 2017 -0500
> +++ b/src/event/ngx_event_openssl.h     Wed Aug 23 21:00:59 2017 -0500
> @@ -172,6 +172,7 @@ ngx_int_t ngx_ssl_session_cache(ngx_ssl_
>  ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl,
>      ngx_array_t *paths);
>  ngx_int_t ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data);
> +ngx_int_t ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
>  ngx_int_t ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c,
>      ngx_uint_t flags);
> 

I would rather place it after the ngx_ssl_ecdh_curve() function, 
similar to how it is placed in the C file.

[...]

-- 
Maxim Dounin
http://nginx.org/

From mdounin at mdounin.ru  Thu Aug 31 14:44:23 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 31 Aug 2017 17:44:23 +0300
Subject: [PATCH] [PATCH 3 of 4] SSL: add PSK identity variable
In-Reply-To: <a11e114a2bcde4afb515.1503541337@OLA-6J5NNS1.ad.garmin.com>
References: <010420>
 <a11e114a2bcde4afb515.1503541337@OLA-6J5NNS1.ad.garmin.com>
Message-ID: <20170831144423.GF93611@mdounin.ru>

Hello!

On Wed, Aug 23, 2017 at 09:22:17PM -0500, Nate Karstens wrote:

> # HG changeset patch
> # User Nate Karstens <nate.karstens at garmin.com>
> # Date 1503540211 18000
> #      Wed Aug 23 21:03:31 2017 -0500
> # Node ID a11e114a2bcde4afb515dd0b70f3ef39693f475a
> # Parent  97953fe374455a04973268c4b2fbadd7ced91ffe
> [PATCH 3 of 4] SSL: add PSK identity variable.

Same as in previous patches, there is no need for "[PATCH ...".

> 
> Adds the variable $ssl_psk_identity to get the PSK identity
> used in a connnection secured with a PSK cipher suite.
> 
> Signed-off-by: Nate Karstens <nate.karstens at garmin.com>
> 
> diff -r 97953fe37445 -r a11e114a2bcd src/event/ngx_event_openssl.c
> --- a/src/event/ngx_event_openssl.c     Wed Aug 23 21:00:59 2017 -0500
> +++ b/src/event/ngx_event_openssl.c     Wed Aug 23 21:03:31 2017 -0500
> @@ -4388,6 +4388,38 @@ ngx_ssl_parse_time(
>  }
> 
> 
> +ngx_int_t
> +ngx_ssl_get_psk_identity(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s)
> +{
> +#ifdef PSK_MAX_IDENTITY_LEN
> +
> +    const char  *identity;
> +
> +    s->len = 0;
> +
> +    identity = SSL_get_psk_identity(c->ssl->connection);
> +
> +    if (identity) {
> +        s->len = ngx_strlen(identity);
> +
> +        s->data = ngx_pnalloc(pool, s->len + 1);

There is no need to allocate space for and/or copy terminating NUL 
character.  In nginx, ngx_str_t strings are not null-terminated 
unless it is required for some reason (for example, in file 
names).

> +        if (s->data == NULL) {
> +            return NGX_ERROR;
> +        }
> +
> +        ngx_cpystrn(s->data, (u_char *) identity, s->len + 1);
> +    }
> +
> +#else
> +
> +    s->len = 0;
> +
> +#endif

Taking "s->len = 0" out of the #ifdef should produce slightly more 
readable (and shorter code) code.  Using recent 
ngx_ssl_get_server_name() code with corresponding modifications:

ngx_int_t
ngx_ssl_get_psk_identity(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s)
{
#ifdef PSK_MAX_IDENTITY_LEN
    
    size_t       len;
    const char  *identity;
    
    identity = SSL_get_psk_identity(c->ssl->connection);
    
    if (identity) {
        len = ngx_strlen(identity);
    
        s->len = len;
        s->data = ngx_pnalloc(pool, len);
        if (s->data == NULL) {
            return NGX_ERROR;
        }

        ngx_memcpy(s->data, identity, len);
    
        return NGX_OK;
    }

#endif

    s->len = 0;
    return NGX_OK;
}

[...]

-- 
Maxim Dounin
http://nginx.org/

From mdounin at mdounin.ru  Thu Aug 31 14:44:44 2017
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 31 Aug 2017 17:44:44 +0300
Subject: [PATCH] [PATCH 4 of 4] SSL: add identity hint config directive
In-Reply-To: <62b4032371bd45217d40.1503541363@OLA-6J5NNS1.ad.garmin.com>
References: <010421>
 <62b4032371bd45217d40.1503541363@OLA-6J5NNS1.ad.garmin.com>
Message-ID: <20170831144444.GG93611@mdounin.ru>

Hello!

On Wed, Aug 23, 2017 at 09:22:43PM -0500, Nate Karstens wrote:

> # HG changeset patch
> # User Nate Karstens <nate.karstens at garmin.com>
> # Date 1503540237 18000
> #      Wed Aug 23 21:03:57 2017 -0500
> # Node ID 62b4032371bd45217d40e2f0daf8ecd6956601d8
> # Parent  a11e114a2bcde4afb515dd0b70f3ef39693f475a
> [PATCH 4 of 4] SSL: add identity hint config directive.

As in previous patches, there should be no "[PATCH 4 of 4] ".
Otherwise loos good.

Following this and previous patches review, here are all four 
patches adjusted according to the comments.  Please take a look if 
it works for you.

# HG changeset patch
# User Nate Karstens <nate.karstens at garmin.com>
# Date 1503540018 18000
#      Wed Aug 23 21:00:18 2017 -0500
# Node ID a87e224e8d6b2993dfcd8903bfb0e7eb7fd934fa
# Parent  c7d4017c8876af6d8570e400320537d7d39e9578
Core: add function to decode hexadecimal strings.

Adds functionality to convert a hexadecimal string into binary data.
This will be used to decode PSKs stored in hexadecimal representation.

Signed-off-by: Nate Karstens <nate.karstens at garmin.com>

diff --git a/src/core/ngx_string.c b/src/core/ngx_string.c
--- a/src/core/ngx_string.c
+++ b/src/core/ngx_string.c
@@ -1118,6 +1118,56 @@ ngx_hex_dump(u_char *dst, u_char *src, s
 }
 
 
+ngx_int_t
+ngx_hex_decode(u_char *dst, u_char *src, size_t len)
+{
+    u_char  ch, decoded;
+
+    if (len & 1) {
+        return NGX_ERROR;
+    }
+
+    while (len) {
+        ch = *src++;
+        len -= 2;
+
+        if (ch >= '0' && ch <= '9') {
+            decoded = ch - '0';
+            goto second;
+        }
+
+        ch |= 0x20;
+
+        if (ch >= 'a' && ch <= 'f') {
+            decoded = ch - 'a' + 10;
+            goto second;
+        }
+
+        return NGX_ERROR;
+
+    second:
+
+        ch = *src++;
+
+        if (ch >= '0' && ch <= '9') {
+            *dst++ = (u_char) ((decoded << 4) + ch - '0');
+            continue;
+        }
+
+        ch |= 0x20;
+
+        if (ch >= 'a' && ch <= 'f') {
+            *dst++ = (u_char) ((decoded << 4) + ch - 'a' + 10);
+            continue;
+        }
+
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+
 void
 ngx_encode_base64(ngx_str_t *dst, ngx_str_t *src)
 {
diff --git a/src/core/ngx_string.h b/src/core/ngx_string.h
--- a/src/core/ngx_string.h
+++ b/src/core/ngx_string.h
@@ -177,6 +177,7 @@ time_t ngx_atotm(u_char *line, size_t n)
 ngx_int_t ngx_hextoi(u_char *line, size_t n);
 
 u_char *ngx_hex_dump(u_char *dst, u_char *src, size_t len);
+ngx_int_t ngx_hex_decode(u_char *dst, u_char *src, size_t len);
 
 
 #define ngx_base64_encoded_length(len)  (((len + 2) / 3) * 4)
# HG changeset patch
# User Nate Karstens <nate.karstens at garmin.com>
# Date 1503540059 18000
#      Wed Aug 23 21:00:59 2017 -0500
# Node ID d89f77108fa8a20bc1fb9cdbaf43fefbc5e07119
# Parent  a87e224e8d6b2993dfcd8903bfb0e7eb7fd934fa
SSL: add support for PSK cipher suites.

Adds support for TLS connections using PSK cipher suites. A new
configuration directive, ssl_psk_file, specifies the file that
contains a list of identities and associated PSKs. Each line of
the file begins with the identity, followed by a colon character
(':'), and ending with the PSK. As required by RFC 4279 section
5.4, PSKs may be entered either as plain text or using hexadecimal
encoding. Hexadecimal PSKs must begin with "{HEX}". PSKs without
this prefix are assumed to be plain text, but they may optionally
begin with "{PLAIN}" to denote this. Some examples:

gary:plain_text_password
min:{PLAIN}another_text_password
cliff:{HEX}ab0123CD

PSK functionality can be easily tested with the OpenSSL s_client
using the "-psk" and "-psk_identity" options.

Signed-off-by: Nate Karstens <nate.karstens at garmin.com>

diff --git a/contrib/vim/syntax/nginx.vim b/contrib/vim/syntax/nginx.vim
--- a/contrib/vim/syntax/nginx.vim
+++ b/contrib/vim/syntax/nginx.vim
@@ -550,6 +550,7 @@ syn keyword ngxDirective contained ssl_p
 syn keyword ngxDirective contained ssl_prefer_server_ciphers
 syn keyword ngxDirective contained ssl_preread
 syn keyword ngxDirective contained ssl_protocols
+syn keyword ngxDirective contained ssl_psk_file
 syn keyword ngxDirective contained ssl_session_cache
 syn keyword ngxDirective contained ssl_session_ticket_key
 syn keyword ngxDirective contained ssl_session_tickets
diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -11,6 +11,7 @@
 
 
 #define NGX_SSL_PASSWORD_BUFFER_SIZE  4096
+#define NGX_SSL_PSK_BUFFER_SIZE       4096
 
 
 typedef struct {
@@ -24,6 +25,10 @@ static int ngx_ssl_verify_callback(int o
 static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where,
     int ret);
 static void ngx_ssl_passwords_cleanup(void *data);
+#ifdef PSK_MAX_IDENTITY_LEN
+static unsigned int ngx_ssl_psk_callback(ngx_ssl_conn_t *ssl_conn,
+    const char *identity, unsigned char *psk, unsigned int max_psk_len);
+#endif
 static void ngx_ssl_handshake_handler(ngx_event_t *ev);
 static ngx_int_t ngx_ssl_handle_recv(ngx_connection_t *c, int n);
 static void ngx_ssl_write_handler(ngx_event_t *wev);
@@ -110,6 +115,7 @@ int  ngx_ssl_connection_index;
 int  ngx_ssl_server_conf_index;
 int  ngx_ssl_session_cache_index;
 int  ngx_ssl_session_ticket_keys_index;
+int  ngx_ssl_psk_index;
 int  ngx_ssl_certificate_index;
 int  ngx_ssl_next_certificate_index;
 int  ngx_ssl_certificate_name_index;
@@ -195,6 +201,14 @@ ngx_ssl_init(ngx_log_t *log)
         return NGX_ERROR;
     }
 
+    ngx_ssl_psk_index = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL);
+
+    if (ngx_ssl_psk_index == -1) {
+        ngx_ssl_error(NGX_LOG_ALERT, log, 0,
+                      "SSL_CTX_get_ex_new_index() failed");
+        return NGX_ERROR;
+    }
+
     ngx_ssl_certificate_index = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL,
                                                          NULL);
     if (ngx_ssl_certificate_index == -1) {
@@ -1163,6 +1177,170 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_s
 
 
 ngx_int_t
+ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file)
+{
+#ifdef PSK_MAX_IDENTITY_LEN
+
+    if (ngx_conf_full_name(cf->cycle, file, 1) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_psk_index, file) == 0) {
+        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
+                      "SSL_CTX_set_ex_data() failed");
+        return NGX_ERROR;
+    }
+
+    SSL_CTX_set_psk_server_callback(ssl->ctx, ngx_ssl_psk_callback);
+
+#endif
+
+    return NGX_OK;
+}
+
+
+#ifdef PSK_MAX_IDENTITY_LEN
+
+static unsigned int
+ngx_ssl_psk_callback(ngx_ssl_conn_t *ssl_conn, const char *identity,
+    unsigned char *psk, unsigned int max_psk_len)
+{
+    u_char            *p, *last, *end, *colon;
+    size_t             len;
+    ssize_t            n;
+    SSL_CTX           *ssl_ctx;
+    ngx_fd_t           fd;
+    ngx_str_t         *file;
+    unsigned int       psk_len;
+    ngx_connection_t  *c;
+    u_char             buf[NGX_SSL_PSK_BUFFER_SIZE];
+
+    c = ngx_ssl_get_connection(ssl_conn);
+
+    ssl_ctx = SSL_get_SSL_CTX(ssl_conn);
+    file = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_psk_index);
+
+    fd = ngx_open_file(file->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
+    if (fd == NGX_INVALID_FILE) {
+        ngx_ssl_error(NGX_LOG_ERR, c->log, ngx_errno,
+                      ngx_open_file_n " \"%V\" failed", file);
+        return 0;
+    }
+
+    psk_len = 0;
+
+    len = 0;
+    last = buf;
+
+    do {
+        n = ngx_read_fd(fd, last, NGX_SSL_PSK_BUFFER_SIZE - len);
+
+        if (n == -1) {
+            ngx_ssl_error(NGX_LOG_ERR, c->log, ngx_errno,
+                          ngx_read_fd_n " \"%V\" failed", file);
+            goto cleanup;
+        }
+
+        end = last + n;
+
+        if (len && n == 0) {
+            *end++ = LF;
+        }
+
+        for (p = buf; /* void */; p = last) {
+            last = ngx_strlchr(last, end, LF);
+
+            if (last == NULL) {
+                break;
+            }
+
+            len = last++ - p;
+
+            if (len && p[len - 1] == CR) {
+                len--;
+            }
+
+            if (len == 0) {
+                continue;
+            }
+
+            colon = ngx_strlchr(p, p + len, ':');
+
+            if (colon == NULL) {
+                continue;
+            }
+
+            *colon = '\0';
+
+            if (ngx_strcmp(p, identity) != 0) {
+                continue;
+            }
+
+            len -= colon + 1 - p;
+            p = colon + 1;
+
+            if (ngx_strncmp(p, "{HEX}", sizeof("{HEX}") - 1) == 0) {
+
+                p += sizeof("{HEX}") - 1;
+                len -= sizeof("{HEX}") - 1;
+
+                if (len / 2 > max_psk_len) {
+                    goto cleanup;
+                }
+
+                if (ngx_hex_decode(psk, p, len) != NGX_OK) {
+                    ngx_memzero(psk, len / 2);
+                    goto cleanup;
+                }
+
+                psk_len = len / 2;
+
+                goto cleanup;
+
+            } else if (ngx_strncmp(p, "{PLAIN}", sizeof("{PLAIN}") - 1) == 0) {
+                p += sizeof("{PLAIN}") - 1;
+                len -= sizeof("{PLAIN}") - 1;
+            }
+
+            if (len > max_psk_len) {
+                goto cleanup;
+            }
+
+            ngx_memcpy(psk, p, len);
+            psk_len = len;
+
+            goto cleanup;
+        }
+
+        len = end - p;
+
+        if (len == NGX_SSL_PSK_BUFFER_SIZE) {
+            ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
+                          "too long line in \"%V\"", file);
+            goto cleanup;
+        }
+
+        ngx_memmove(buf, p, len);
+        last = buf + len;
+
+    } while (n != 0);
+
+cleanup:
+
+    if (ngx_close_file(fd) == NGX_FILE_ERROR) {
+        ngx_ssl_error(NGX_LOG_ALERT, c->log, ngx_errno,
+                      ngx_close_file_n " %V failed", file);
+    }
+
+    ngx_memzero(buf, NGX_SSL_PSK_BUFFER_SIZE);
+
+    return psk_len;
+}
+
+#endif
+
+
+ngx_int_t
 ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags)
 {
     ngx_ssl_connection_t  *sc;
@@ -2071,6 +2249,9 @@ ngx_ssl_connection_error(ngx_connection_
             || n == SSL_R_NO_COMPRESSION_SPECIFIED                   /*  187 */
             || n == SSL_R_NO_SHARED_CIPHER                           /*  193 */
             || n == SSL_R_RECORD_LENGTH_MISMATCH                     /*  213 */
+#ifdef SSL_R_PSK_IDENTITY_NOT_FOUND
+            || n == SSL_R_PSK_IDENTITY_NOT_FOUND                     /*  223 */
+#endif
 #ifdef SSL_R_PARSE_TLSEXT
             || n == SSL_R_PARSE_TLSEXT                               /*  227 */
 #endif
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -167,6 +167,7 @@ RSA *ngx_ssl_rsa512_key_callback(ngx_ssl
 ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file);
 ngx_int_t ngx_ssl_dhparam(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
 ngx_int_t ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *name);
+ngx_int_t ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
 ngx_int_t ngx_ssl_session_cache(ngx_ssl_t *ssl, ngx_str_t *sess_ctx,
     ssize_t builtin_session_cache, ngx_shm_zone_t *shm_zone, time_t timeout);
 ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl,
@@ -253,6 +254,7 @@ extern int  ngx_ssl_connection_index;
 extern int  ngx_ssl_server_conf_index;
 extern int  ngx_ssl_session_cache_index;
 extern int  ngx_ssl_session_ticket_keys_index;
+extern int  ngx_ssl_psk_index;
 extern int  ngx_ssl_certificate_index;
 extern int  ngx_ssl_next_certificate_index;
 extern int  ngx_ssl_certificate_name_index;
diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c
--- a/src/http/modules/ngx_http_ssl_module.c
+++ b/src/http/modules/ngx_http_ssl_module.c
@@ -234,6 +234,13 @@ static ngx_command_t  ngx_http_ssl_comma
       offsetof(ngx_http_ssl_srv_conf_t, stapling_verify),
       NULL },
 
+    { ngx_string("ssl_psk_file"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_str_slot,
+      NGX_HTTP_SRV_CONF_OFFSET,
+      offsetof(ngx_http_ssl_srv_conf_t, psk_file),
+      NULL },
+
       ngx_null_command
 };
 
@@ -543,6 +550,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t 
      *     sscf->shm_zone = NULL;
      *     sscf->stapling_file = { 0, NULL };
      *     sscf->stapling_responder = { 0, NULL };
+     *     sscf->psk_file = { 0, NULL };
      */
 
     sscf->enable = NGX_CONF_UNSET;
@@ -624,6 +632,8 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *
     ngx_conf_merge_str_value(conf->stapling_responder,
                          prev->stapling_responder, "");
 
+    ngx_conf_merge_str_value(conf->psk_file, prev->psk_file, "");
+
     conf->ssl.log = cf->log;
 
     if (conf->enable) {
@@ -804,6 +814,10 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *
 
     }
 
+    if (ngx_ssl_psk_file(cf, &conf->ssl, &conf->psk_file) != NGX_OK) {
+        return NGX_CONF_ERROR;
+    }
+
     return NGX_CONF_OK;
 }
 
diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h
--- a/src/http/modules/ngx_http_ssl_module.h
+++ b/src/http/modules/ngx_http_ssl_module.h
@@ -55,6 +55,8 @@ typedef struct {
     ngx_str_t                       stapling_file;
     ngx_str_t                       stapling_responder;
 
+    ngx_str_t                       psk_file;
+
     u_char                         *file;
     ngx_uint_t                      line;
 } ngx_http_ssl_srv_conf_t;
# HG changeset patch
# User Nate Karstens <nate.karstens at garmin.com>
# Date 1503540211 18000
#      Wed Aug 23 21:03:31 2017 -0500
# Node ID 3d2e75b562a3615757d17a00cc54db2d2e90ddd0
# Parent  d89f77108fa8a20bc1fb9cdbaf43fefbc5e07119
SSL: add PSK identity variable.

Adds the variable $ssl_psk_identity to get the PSK identity
used in a connnection secured with a PSK cipher suite.

Signed-off-by: Nate Karstens <nate.karstens at garmin.com>

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -4350,6 +4350,37 @@ ngx_ssl_parse_time(
 }
 
 
+ngx_int_t
+ngx_ssl_get_psk_identity(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s)
+{
+#ifdef PSK_MAX_IDENTITY_LEN
+
+    size_t       len;
+    const char  *identity;
+
+    identity = SSL_get_psk_identity(c->ssl->connection);
+
+    if (identity) {
+        len = ngx_strlen(identity);
+
+        s->len = len;
+        s->data = ngx_pnalloc(pool, len);
+        if (s->data == NULL) {
+            return NGX_ERROR;
+        }
+
+        ngx_memcpy(s->data, identity, len);
+            
+        return NGX_OK;
+    }
+
+#endif
+
+    s->len = 0;
+    return NGX_OK;
+}
+
+
 static void *
 ngx_openssl_create_conf(ngx_cycle_t *cycle)
 {
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -235,6 +235,8 @@ ngx_int_t ngx_ssl_get_client_v_end(ngx_c
     ngx_str_t *s);
 ngx_int_t ngx_ssl_get_client_v_remain(ngx_connection_t *c, ngx_pool_t *pool,
     ngx_str_t *s);
+ngx_int_t ngx_ssl_get_psk_identity(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
 
 
 ngx_int_t ngx_ssl_handshake(ngx_connection_t *c);
diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c
--- a/src/http/modules/ngx_http_ssl_module.c
+++ b/src/http/modules/ngx_http_ssl_module.c
@@ -340,6 +340,9 @@ static ngx_http_variable_t  ngx_http_ssl
     { ngx_string("ssl_client_v_remain"), NULL, ngx_http_ssl_variable,
       (uintptr_t) ngx_ssl_get_client_v_remain, NGX_HTTP_VAR_CHANGEABLE, 0 },
 
+    { ngx_string("ssl_psk_identity"), NULL, ngx_http_ssl_variable,
+      (uintptr_t) ngx_ssl_get_psk_identity, NGX_HTTP_VAR_CHANGEABLE, 0 },
+
       ngx_http_null_variable
 };
 
# HG changeset patch
# User Nate Karstens <nate.karstens at garmin.com>
# Date 1503540237 18000
#      Wed Aug 23 21:03:57 2017 -0500
# Node ID 3876f3a8d4bb1bdaabc61492bf8be838d9dae5fb
# Parent  3d2e75b562a3615757d17a00cc54db2d2e90ddd0
SSL: add identity hint config directive.

Adds the directive "ssl_psk_identity_hint" to the ngx_http_ssl_module.
This allows the user to specify the PSK identity hint given to the
connecting client.

Signed-off-by: Nate Karstens <nate.karstens at garmin.com>

diff --git a/contrib/vim/syntax/nginx.vim b/contrib/vim/syntax/nginx.vim
--- a/contrib/vim/syntax/nginx.vim
+++ b/contrib/vim/syntax/nginx.vim
@@ -551,6 +551,7 @@ syn keyword ngxDirective contained ssl_p
 syn keyword ngxDirective contained ssl_preread
 syn keyword ngxDirective contained ssl_protocols
 syn keyword ngxDirective contained ssl_psk_file
+syn keyword ngxDirective contained ssl_psk_identity_hint
 syn keyword ngxDirective contained ssl_session_cache
 syn keyword ngxDirective contained ssl_session_ticket_key
 syn keyword ngxDirective contained ssl_session_tickets
diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -1177,7 +1177,8 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_s
 
 
 ngx_int_t
-ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file)
+ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file,
+    ngx_str_t *identity_hint)
 {
 #ifdef PSK_MAX_IDENTITY_LEN
 
@@ -1191,6 +1192,14 @@ ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl
         return NGX_ERROR;
     }
 
+    if (SSL_CTX_use_psk_identity_hint(ssl->ctx, (char *) identity_hint->data)
+        == 0)
+    {
+        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
+                      "SSL_CTX_use_psk_identity_hint() failed");
+        return NGX_ERROR;
+    }
+
     SSL_CTX_set_psk_server_callback(ssl->ctx, ngx_ssl_psk_callback);
 
 #endif
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -167,7 +167,8 @@ RSA *ngx_ssl_rsa512_key_callback(ngx_ssl
 ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file);
 ngx_int_t ngx_ssl_dhparam(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
 ngx_int_t ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *name);
-ngx_int_t ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
+ngx_int_t ngx_ssl_psk_file(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file,
+    ngx_str_t *identity_hint);
 ngx_int_t ngx_ssl_session_cache(ngx_ssl_t *ssl, ngx_str_t *sess_ctx,
     ssize_t builtin_session_cache, ngx_shm_zone_t *shm_zone, time_t timeout);
 ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl,
diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c
--- a/src/http/modules/ngx_http_ssl_module.c
+++ b/src/http/modules/ngx_http_ssl_module.c
@@ -241,6 +241,13 @@ static ngx_command_t  ngx_http_ssl_comma
       offsetof(ngx_http_ssl_srv_conf_t, psk_file),
       NULL },
 
+    { ngx_string("ssl_psk_identity_hint"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_str_slot,
+      NGX_HTTP_SRV_CONF_OFFSET,
+      offsetof(ngx_http_ssl_srv_conf_t, psk_identity_hint),
+      NULL },
+
       ngx_null_command
 };
 
@@ -554,6 +561,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t 
      *     sscf->stapling_file = { 0, NULL };
      *     sscf->stapling_responder = { 0, NULL };
      *     sscf->psk_file = { 0, NULL };
+     *     sscf->psk_identity_hint = { 0, NULL };
      */
 
     sscf->enable = NGX_CONF_UNSET;
@@ -636,6 +644,8 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *
                          prev->stapling_responder, "");
 
     ngx_conf_merge_str_value(conf->psk_file, prev->psk_file, "");
+    ngx_conf_merge_str_value(conf->psk_identity_hint,
+                         prev->psk_identity_hint, "");
 
     conf->ssl.log = cf->log;
 
@@ -817,7 +827,10 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *
 
     }
 
-    if (ngx_ssl_psk_file(cf, &conf->ssl, &conf->psk_file) != NGX_OK) {
+    if (ngx_ssl_psk_file(cf, &conf->ssl, &conf->psk_file,
+                         &conf->psk_identity_hint)
+        != NGX_OK)
+    {
         return NGX_CONF_ERROR;
     }
 
diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h
--- a/src/http/modules/ngx_http_ssl_module.h
+++ b/src/http/modules/ngx_http_ssl_module.h
@@ -56,6 +56,7 @@ typedef struct {
     ngx_str_t                       stapling_responder;
 
     ngx_str_t                       psk_file;
+    ngx_str_t                       psk_identity_hint;
 
     u_char                         *file;
     ngx_uint_t                      line;


-- 
Maxim Dounin
http://nginx.org/

From xeioex at nginx.com  Thu Aug 31 15:38:49 2017
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 31 Aug 2017 15:38:49 +0000
Subject: [njs] Compiling CLI by default if libedit is available.
Message-ID: <hg.37adbd4c8036.1504193929.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/37adbd4c8036
branches:  
changeset: 401:37adbd4c8036
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu Aug 31 18:38:34 2017 +0300
description:
Compiling CLI by default if libedit is available.

diffstat:

 Makefile |  6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)

diffs (16 lines):

diff -r 82dc332cb4c8 -r 37adbd4c8036 Makefile
--- a/Makefile	Tue Aug 29 14:06:23 2017 +0300
+++ b/Makefile	Thu Aug 31 18:38:34 2017 +0300
@@ -7,6 +7,12 @@ NXT_LIB =	nxt
 
 NXT_BUILDDIR =	build
 
+ifneq ($(NXT_EDITLINE_LIB),)
+default: $(NXT_BUILDDIR)/libnjs.a $(NXT_BUILDDIR)/njs
+else
+default: $(NXT_BUILDDIR)/libnjs.a
+endif
+
 $(NXT_BUILDDIR)/libnjs.a: \
 	$(NXT_LIB)/nxt_auto_config.h \
 	$(NXT_BUILDDIR)/njscript.o \

From pluknet at nginx.com  Thu Aug 31 17:14:41 2017
From: pluknet at nginx.com (Sergey Kandaurov)
Date: Thu, 31 Aug 2017 17:14:41 +0000
Subject: [njs] Backed out changeset 37adbd4c8036, reimplemented properly.
Message-ID: <hg.46ea39d059e1.1504199681.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/46ea39d059e1
branches:  
changeset: 402:46ea39d059e1
user:      Sergey Kandaurov <pluknet at nginx.com>
date:      Thu Aug 31 20:03:13 2017 +0300
description:
Backed out changeset 37adbd4c8036, reimplemented properly.

Changeset 37adbd4c8036 breaks build with bmake by introducing Gnu constructs.
Instead, make CLI compilation dependent on libedit by conditionally extending
the default target.

diffstat:

 Makefile          |  6 ------
 nxt/auto/editline |  2 ++
 2 files changed, 2 insertions(+), 6 deletions(-)

diffs (28 lines):

diff -r 37adbd4c8036 -r 46ea39d059e1 Makefile
--- a/Makefile	Thu Aug 31 18:38:34 2017 +0300
+++ b/Makefile	Thu Aug 31 20:03:13 2017 +0300
@@ -7,12 +7,6 @@ NXT_LIB =	nxt
 
 NXT_BUILDDIR =	build
 
-ifneq ($(NXT_EDITLINE_LIB),)
-default: $(NXT_BUILDDIR)/libnjs.a $(NXT_BUILDDIR)/njs
-else
-default: $(NXT_BUILDDIR)/libnjs.a
-endif
-
 $(NXT_BUILDDIR)/libnjs.a: \
 	$(NXT_LIB)/nxt_auto_config.h \
 	$(NXT_BUILDDIR)/njscript.o \
diff -r 37adbd4c8036 -r 46ea39d059e1 nxt/auto/editline
--- a/nxt/auto/editline	Thu Aug 31 18:38:34 2017 +0300
+++ b/nxt/auto/editline	Thu Aug 31 20:03:13 2017 +0300
@@ -32,6 +32,8 @@ if [ $nxt_found = yes ]; then
 
 NXT_EDITLINE_CFLAGS = $nxt_feature_incs
 NXT_EDITLINE_LIB = $nxt_feature_libs
+
+default:	njs
 END
 
 else

From xeioex at nginx.com  Thu Aug 31 17:28:13 2017
From: xeioex at nginx.com (Dmitry Volyntsev)
Date: Thu, 31 Aug 2017 17:28:13 +0000
Subject: [njs] Interactive shell: console object.
Message-ID: <hg.27aa477208f5.1504200493.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/27aa477208f5
branches:  
changeset: 403:27aa477208f5
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu Aug 31 20:27:31 2017 +0300
description:
Interactive shell: console object.

diffstat:

 nginx/ngx_http_js_module.c   |    2 +-
 nginx/ngx_stream_js_module.c |    2 +-
 njs/njs.c                    |  154 +++++++++++++++++++++++++++++++++++++++++-
 njs/njs_builtin.c            |  103 ++++++++++++++++++++++------
 njs/njs_builtin.h            |   15 ++++
 njs/njscript.c               |    8 +-
 njs/njscript.h               |    3 +-
 njs/test/njs_unit_test.c     |    2 +-
 8 files changed, 255 insertions(+), 34 deletions(-)

diffs (558 lines):

diff -r 46ea39d059e1 -r 27aa477208f5 nginx/ngx_http_js_module.c
--- a/nginx/ngx_http_js_module.c	Thu Aug 31 20:03:13 2017 +0300
+++ b/nginx/ngx_http_js_module.c	Thu Aug 31 20:27:31 2017 +0300
@@ -1322,7 +1322,7 @@ ngx_http_js_include(ngx_conf_t *cf, ngx_
     ngx_memzero(&options, sizeof(njs_vm_opt_t));
 
     options.mcp = mcp;
-    options.externals = &externals;
+    options.externals_hash = &externals;
 
     jlcf->vm = njs_vm_create(&options);
     if (jlcf->vm == NULL) {
diff -r 46ea39d059e1 -r 27aa477208f5 nginx/ngx_stream_js_module.c
--- a/nginx/ngx_stream_js_module.c	Thu Aug 31 20:03:13 2017 +0300
+++ b/nginx/ngx_stream_js_module.c	Thu Aug 31 20:27:31 2017 +0300
@@ -1032,7 +1032,7 @@ ngx_stream_js_include(ngx_conf_t *cf, ng
     ngx_memzero(&options, sizeof(njs_vm_opt_t));
 
     options.mcp = mcp;
-    options.externals = &externals;
+    options.externals_hash = &externals;
 
     jscf->vm = njs_vm_create(&options);
     if (jscf->vm == NULL) {
diff -r 46ea39d059e1 -r 27aa477208f5 njs/njs.c
--- a/njs/njs.c	Thu Aug 31 20:03:13 2017 +0300
+++ b/njs/njs.c	Thu Aug 31 20:27:31 2017 +0300
@@ -21,9 +21,12 @@
 #include <nxt_array.h>
 #include <nxt_lvlhsh.h>
 #include <nxt_random.h>
+#include <nxt_djb_hash.h>
 #include <nxt_mem_cache_pool.h>
 #include <njscript.h>
 #include <njs_vm.h>
+#include <njs_object.h>
+#include <njs_builtin.h>
 #include <njs_variable.h>
 #include <njs_parser.h>
 
@@ -54,6 +57,7 @@ typedef struct {
 
 
 static nxt_int_t njs_get_options(njs_opts_t *opts, int argc, char **argv);
+static nxt_int_t njs_externals_init(njs_opts_t *opts, njs_vm_opt_t *vm_options);
 static nxt_int_t njs_interactive_shell(njs_opts_t *opts,
     njs_vm_opt_t *vm_options);
 static nxt_int_t njs_process_file(njs_opts_t *opts, njs_vm_opt_t *vm_options);
@@ -64,7 +68,56 @@ static nxt_int_t njs_editline_init(njs_v
 static char **njs_completion_handler(const char *text, int start, int end);
 static char *njs_completion_generator(const char *text, int state);
 
+static njs_ret_t njs_ext_console_log(njs_vm_t *vm, njs_value_t *args,
+    nxt_uint_t nargs, njs_index_t unused);
+static njs_ret_t njs_ext_console_help(njs_vm_t *vm, njs_value_t *args,
+    nxt_uint_t nargs, njs_index_t unused);
 
+
+static njs_external_t  njs_ext_console[] = {
+
+    { nxt_string("log"),
+      NJS_EXTERN_METHOD,
+      NULL,
+      0,
+      NULL,
+      NULL,
+      NULL,
+      NULL,
+      NULL,
+      njs_ext_console_log,
+      0 },
+
+    { nxt_string("help"),
+      NJS_EXTERN_METHOD,
+      NULL,
+      0,
+      NULL,
+      NULL,
+      NULL,
+      NULL,
+      NULL,
+      njs_ext_console_help,
+      0 },
+};
+
+static njs_external_t  njs_externals[] = {
+
+    { nxt_string("console"),
+      NJS_EXTERN_OBJECT,
+      njs_ext_console,
+      nxt_nitems(njs_ext_console),
+      NULL,
+      NULL,
+      NULL,
+      NULL,
+      NULL,
+      NULL,
+      0 },
+};
+
+
+static nxt_lvlhsh_t      njs_externals_hash;
 static njs_completion_t  njs_completion;
 
 
@@ -96,6 +149,10 @@ main(int argc, char **argv)
     vm_options.accumulative = 1;
     vm_options.backtrace = 1;
 
+    if (njs_externals_init(&opts, &vm_options) != NXT_OK) {
+        return EXIT_FAILURE;
+    }
+
     if (opts.interactive) {
         ret = njs_interactive_shell(&opts, &vm_options);
 
@@ -150,6 +207,36 @@ njs_get_options(njs_opts_t *opts, int ar
 
 
 static nxt_int_t
+njs_externals_init(njs_opts_t *opts, njs_vm_opt_t *vm_options)
+{
+    void        **ext;
+    nxt_uint_t  i;
+
+    nxt_lvlhsh_init(&njs_externals_hash);
+
+    for (i = 0; i < nxt_nitems(njs_externals); i++) {
+        if (njs_vm_external_add(&njs_externals_hash, vm_options->mcp,
+                                (uintptr_t) i, &njs_externals[i], 1)
+            != NXT_OK)
+        {
+            fprintf(stderr, "could not add external objects\n");
+            return NXT_ERROR;
+        }
+    }
+
+    ext = nxt_mem_cache_zalloc(vm_options->mcp, sizeof(void *) * i);
+    if (ext == NULL) {
+        return NXT_ERROR;
+    }
+
+    vm_options->external = ext;
+    vm_options->externals_hash = &njs_externals_hash;
+
+    return NXT_OK;
+}
+
+
+static nxt_int_t
 njs_interactive_shell(njs_opts_t *opts, njs_vm_opt_t *vm_options)
 {
     njs_vm_t     *vm;
@@ -168,7 +255,11 @@ njs_interactive_shell(njs_opts_t *opts, 
         return NXT_ERROR;
     }
 
-    printf("interactive njscript\n");
+    printf("interactive njscript\n\n");
+
+    printf("v<Tab> -> the properties of v object.\n");
+    printf("v.<Tab> -> all the available prototype methods.\n");
+    printf("type console.help() for more information\n\n");
 
     for ( ;; ) {
         line.start = (u_char *) readline(">> ");
@@ -185,8 +276,8 @@ njs_interactive_shell(njs_opts_t *opts, 
 
         ret = njs_process_script(vm, opts, &line, &out);
         if (ret != NXT_OK) {
-            printf("njs_process_script() failed\n");
-            return NXT_ERROR;
+            printf("shell: failed to get retval from VM\n");
+            continue;
         }
 
         printf("%.*s\n", (int) out.length, out.start);
@@ -290,7 +381,7 @@ njs_process_file(njs_opts_t *opts, njs_v
 
     ret = njs_process_script(vm, opts, &script, &out);
     if (ret != NXT_OK) {
-        fprintf(stderr, "njs_process_script() failed\n");
+        fprintf(stderr, "failed to get retval from VM\n");
         return NXT_ERROR;
     }
 
@@ -498,3 +589,58 @@ njs_completion_generator(const char *tex
 
     return NULL;
 }
+
+
+static njs_ret_t
+njs_ext_console_log(njs_vm_t *vm, njs_value_t *args, nxt_uint_t nargs,
+    njs_index_t unused)
+{
+    nxt_str_t  msg;
+
+    msg.length = 0;
+
+    if (nargs >= 2
+        && njs_value_to_ext_string(vm, &msg, njs_argument(args, 1))
+           == NJS_ERROR)
+    {
+
+        return NJS_ERROR;
+    }
+
+    printf("%.*s\n", (int) msg.length, msg.start);
+
+    vm->retval = njs_value_void;
+
+    return NJS_OK;
+}
+
+
+static njs_ret_t
+njs_ext_console_help(njs_vm_t *vm, njs_value_t *args, nxt_uint_t nargs,
+    njs_index_t unused)
+{
+    nxt_uint_t  i;
+
+    printf("VM built-in objects:\n");
+    for (i = NJS_CONSTRUCTOR_OBJECT; i < NJS_CONSTRUCTOR_MAX; i++) {
+        printf("  %.*s\n", (int) njs_constructor_init[i]->name.length,
+               njs_constructor_init[i]->name.start);
+    }
+
+    for (i = NJS_OBJECT_THIS; i < NJS_OBJECT_MAX; i++) {
+        if (njs_object_init[i] != NULL) {
+            printf("  %.*s\n", (int) njs_object_init[i]->name.length,
+                   njs_object_init[i]->name.start);
+        }
+    }
+
+    printf("\nEmbedded objects:\n");
+    for (i = 0; i < nxt_nitems(njs_externals); i++) {
+        printf("  %.*s\n", (int) njs_externals[i].name.length,
+               njs_externals[i].name.start);
+    }
+
+    printf("\n");
+
+    return NJS_OK;
+}
diff -r 46ea39d059e1 -r 27aa477208f5 njs/njs_builtin.c
--- a/njs/njs_builtin.c	Thu Aug 31 20:03:13 2017 +0300
+++ b/njs/njs_builtin.c	Thu Aug 31 20:27:31 2017 +0300
@@ -22,6 +22,7 @@
 #include <njs_array.h>
 #include <njs_function.h>
 #include <njs_variable.h>
+#include <njs_extern.h>
 #include <njs_parser.h>
 #include <njs_regexp.h>
 #include <njs_date.h>
@@ -40,13 +41,13 @@ static nxt_int_t njs_builtin_completions
     const char **completions);
 
 
-static const njs_object_init_t    *object_init[] = {
+const njs_object_init_t    *njs_object_init[] = {
     NULL,                         /* global this        */
     &njs_math_object_init,        /* Math               */
 };
 
 
-static const njs_object_init_t  *prototype_init[] = {
+const njs_object_init_t  *njs_prototype_init[] = {
     &njs_object_prototype_init,
     &njs_array_prototype_init,
     &njs_boolean_prototype_init,
@@ -58,7 +59,7 @@ static const njs_object_init_t  *prototy
 };
 
 
-static const njs_object_init_t    *constructor_init[] = {
+const njs_object_init_t    *njs_constructor_init[] = {
     &njs_object_constructor_init,
     &njs_array_constructor_init,
     &njs_boolean_constructor_init,
@@ -189,10 +190,10 @@ njs_builtin_objects_create(njs_vm_t *vm)
     objects = vm->shared->objects;
 
     for (i = NJS_OBJECT_THIS; i < NJS_OBJECT_MAX; i++) {
-        if (object_init[i] != NULL) {
+        if (njs_object_init[i] != NULL) {
             ret = njs_object_hash_create(vm, &objects[i].shared_hash,
-                                         object_init[i]->properties,
-                                         object_init[i]->items);
+                                         njs_object_init[i]->properties,
+                                         njs_object_init[i]->items);
             if (nxt_slow_path(ret != NXT_OK)) {
                 return NXT_ERROR;
             }
@@ -231,8 +232,8 @@ njs_builtin_objects_create(njs_vm_t *vm)
         prototypes[i] = prototype_values[i];
 
         ret = njs_object_hash_create(vm, &prototypes[i].object.shared_hash,
-                                     prototype_init[i]->properties,
-                                     prototype_init[i]->items);
+                                     njs_prototype_init[i]->properties,
+                                     njs_prototype_init[i]->items);
         if (nxt_slow_path(ret != NXT_OK)) {
             return NXT_ERROR;
         }
@@ -257,8 +258,8 @@ njs_builtin_objects_create(njs_vm_t *vm)
         constructors[i].args_types[4] = native_constructors[i].args_types[4];
 
         ret = njs_object_hash_create(vm, &constructors[i].object.shared_hash,
-                                     constructor_init[i]->properties,
-                                     constructor_init[i]->items);
+                                     njs_constructor_init[i]->properties,
+                                     njs_constructor_init[i]->items);
         if (nxt_slow_path(ret != NXT_OK)) {
             return NXT_ERROR;
         }
@@ -375,11 +376,12 @@ njs_builtin_completions(njs_vm_t *vm, si
     size_t                  n, len;
     nxt_str_t               string;
     nxt_uint_t              i, k;
+    njs_extern_t            *ext_object, *ext_prop;
     njs_object_t            *objects;
     njs_keyword_t           *keyword;
     njs_function_t          *constructors;
     njs_object_prop_t       *prop;
-    nxt_lvlhsh_each_t       lhe;
+    nxt_lvlhsh_each_t       lhe, lhe_prop;
     njs_object_prototype_t  *prototypes;
 
     n = 0;
@@ -404,7 +406,7 @@ njs_builtin_completions(njs_vm_t *vm, si
     objects = vm->shared->objects;
 
     for (i = NJS_OBJECT_THIS; i < NJS_OBJECT_MAX; i++) {
-        if (object_init[i] == NULL) {
+        if (njs_object_init[i] == NULL) {
             continue;
         }
 
@@ -419,14 +421,14 @@ njs_builtin_completions(njs_vm_t *vm, si
 
             if (completions != NULL) {
                 njs_string_get(&prop->name, &string);
-                len = object_init[i]->name.length + string.length + 2;
+                len = njs_object_init[i]->name.length + string.length + 2;
 
                 compl = nxt_mem_cache_zalloc(vm->mem_cache_pool, len);
                 if (compl == NULL) {
                     return NXT_ERROR;
                 }
 
-                snprintf(compl, len, "%s.%s", object_init[i]->name.start,
+                snprintf(compl, len, "%s.%s", njs_object_init[i]->name.start,
                          string.start);
 
                 completions[n++] = (char *) compl;
@@ -490,15 +492,68 @@ njs_builtin_completions(njs_vm_t *vm, si
 
             if (completions != NULL) {
                 njs_string_get(&prop->name, &string);
-                len = constructor_init[i]->name.length + string.length + 2;
+                len = njs_constructor_init[i]->name.length + string.length + 2;
 
                 compl = nxt_mem_cache_zalloc(vm->mem_cache_pool, len);
                 if (compl == NULL) {
                     return NXT_ERROR;
                 }
 
-                snprintf(compl, len, "%s.%s", constructor_init[i]->name.start,
-                         string.start);
+                snprintf(compl, len, "%s.%s",
+                         njs_constructor_init[i]->name.start, string.start);
+
+                completions[n++] = (char *) compl;
+
+            } else {
+                n++;
+            }
+        }
+    }
+
+    nxt_lvlhsh_each_init(&lhe, &njs_extern_hash_proto);
+
+    for ( ;; ) {
+        ext_object = nxt_lvlhsh_each(&vm->externals_hash, &lhe);
+
+        if (ext_object == NULL) {
+            break;
+        }
+
+        nxt_lvlhsh_each_init(&lhe_prop, &njs_extern_hash_proto);
+
+        if (completions != NULL) {
+            len = ext_object->name.length + 1;
+            compl = nxt_mem_cache_zalloc(vm->mem_cache_pool, len);
+            if (compl == NULL) {
+                return NXT_ERROR;
+            }
+
+            snprintf(compl, len, "%.*s",
+                     (int) ext_object->name.length, ext_object->name.start);
+
+            completions[n++] = (char *) compl;
+
+        } else {
+            n++;
+        }
+
+        for ( ;; ) {
+            ext_prop = nxt_lvlhsh_each(&ext_object->hash, &lhe_prop);
+
+            if (ext_prop == NULL) {
+                break;
+            }
+
+            if (completions != NULL) {
+                len = ext_object->name.length + ext_prop->name.length + 2;
+                compl = nxt_mem_cache_zalloc(vm->mem_cache_pool, len);
+                if (compl == NULL) {
+                    return NXT_ERROR;
+                }
+
+                snprintf(compl, len, "%.*s.%.*s",
+                         (int) ext_object->name.length, ext_object->name.start,
+                         (int) ext_prop->name.length, ext_prop->name.start);
 
                 completions[n++] = (char *) compl;
 
@@ -533,7 +588,7 @@ njs_builtin_match_native_function(njs_vm
     objects = vm->shared->objects;
 
     for (i = NJS_OBJECT_THIS; i < NJS_OBJECT_MAX; i++) {
-        if (object_init[i] == NULL) {
+        if (njs_object_init[i] == NULL) {
             continue;
         }
 
@@ -552,7 +607,7 @@ njs_builtin_match_native_function(njs_vm
 
             if (function == prop->value.data.u.function) {
                 njs_string_get(&prop->name, &string);
-                len = object_init[i]->name.length + string.length
+                len = njs_object_init[i]->name.length + string.length
                       + sizeof(".");
 
                 buf = nxt_mem_cache_zalloc(vm->mem_cache_pool, len);
@@ -560,7 +615,7 @@ njs_builtin_match_native_function(njs_vm
                     return NXT_ERROR;
                 }
 
-                snprintf(buf, len, "%s.%s", object_init[i]->name.start,
+                snprintf(buf, len, "%s.%s", njs_object_init[i]->name.start,
                          string.start);
 
                 name->length = len;
@@ -589,7 +644,7 @@ njs_builtin_match_native_function(njs_vm
 
             if (function == prop->value.data.u.function) {
                 njs_string_get(&prop->name, &string);
-                len = prototype_init[i]->name.length + string.length
+                len = njs_prototype_init[i]->name.length + string.length
                       + sizeof(".prototype.");
 
                 buf = nxt_mem_cache_zalloc(vm->mem_cache_pool, len);
@@ -598,7 +653,7 @@ njs_builtin_match_native_function(njs_vm
                 }
 
                 snprintf(buf, len, "%s.prototype.%s",
-                         prototype_init[i]->name.start, string.start);
+                         njs_prototype_init[i]->name.start, string.start);
 
                 name->length = len;
                 name->start = (u_char *) buf;
@@ -626,7 +681,7 @@ njs_builtin_match_native_function(njs_vm
 
             if (function == prop->value.data.u.function) {
                 njs_string_get(&prop->name, &string);
-                len = constructor_init[i]->name.length + string.length
+                len = njs_constructor_init[i]->name.length + string.length
                       + sizeof(".");
 
                 buf = nxt_mem_cache_zalloc(vm->mem_cache_pool, len);
@@ -634,7 +689,7 @@ njs_builtin_match_native_function(njs_vm
                     return NXT_ERROR;
                 }
 
-                snprintf(buf, len, "%s.%s", constructor_init[i]->name.start,
+                snprintf(buf, len, "%s.%s", njs_constructor_init[i]->name.start,
                          string.start);
 
                 name->length = len;
diff -r 46ea39d059e1 -r 27aa477208f5 njs/njs_builtin.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/njs/njs_builtin.h	Thu Aug 31 20:27:31 2017 +0300
@@ -0,0 +1,15 @@
+
+/*
+ * Copyright (C) Dmitry Volyntsev
+ * Copyright (C) NGINX, Inc.
+ */
+
+#ifndef _NJS_BUILTIN_H_INCLUDED_
+#define _NJS_BUILTIN_H_INCLUDED_
+
+
+extern const njs_object_init_t  *njs_object_init[];
+extern const njs_object_init_t  *njs_prototype_init[];
+extern const njs_object_init_t  *njs_constructor_init[];
+
+#endif /* _NJS_BUILTIN_H_INCLUDED_ */
diff -r 46ea39d059e1 -r 27aa477208f5 njs/njscript.c
--- a/njs/njscript.c	Thu Aug 31 20:03:13 2017 +0300
+++ b/njs/njscript.c	Thu Aug 31 20:27:31 2017 +0300
@@ -162,12 +162,16 @@ njs_vm_create(njs_vm_opt_t *options)
             if (nxt_slow_path(ret != NXT_OK)) {
                 return NULL;
             }
+
+            if (options->externals_hash != NULL) {
+                vm->external = options->external;
+            }
         }
 
         nxt_lvlhsh_init(&vm->values_hash);
 
-        if (options->externals != NULL) {
-            vm->externals_hash = *options->externals;
+        if (options->externals_hash != NULL) {
+            vm->externals_hash = *options->externals_hash;
         }
 
         vm->trace.level = NXT_LEVEL_TRACE;
diff -r 46ea39d059e1 -r 27aa477208f5 njs/njscript.h
--- a/njs/njscript.h	Thu Aug 31 20:03:13 2017 +0300
+++ b/njs/njscript.h	Thu Aug 31 20:27:31 2017 +0300
@@ -66,7 +66,8 @@ struct njs_external_s {
 };
 
 typedef struct {
-    nxt_lvlhsh_t                    *externals;
+    void                            **external;
+    nxt_lvlhsh_t                    *externals_hash;
     njs_vm_shared_t                 *shared;
     nxt_mem_cache_pool_t            *mcp;
 
diff -r 46ea39d059e1 -r 27aa477208f5 njs/test/njs_unit_test.c
--- a/njs/test/njs_unit_test.c	Thu Aug 31 20:03:13 2017 +0300
+++ b/njs/test/njs_unit_test.c	Thu Aug 31 20:27:31 2017 +0300
@@ -8170,7 +8170,7 @@ njs_unit_test(nxt_bool_t disassemble)
         memset(&options, 0, sizeof(njs_vm_opt_t));
 
         options.mcp = mcp;
-        options.externals = &externals;
+        options.externals_hash = &externals;
 
         vm = njs_vm_create(&options);
         if (vm == NULL) {

From igor at sysoev.ru  Thu Aug 31 17:50:51 2017
From: igor at sysoev.ru (Igor Sysoev)
Date: Thu, 31 Aug 2017 17:50:51 +0000
Subject: [njs] Version 0.1.13.
Message-ID: <hg.d548b78eb881.1504201851.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/d548b78eb881
branches:  
changeset: 404:d548b78eb881
user:      Igor Sysoev <igor at sysoev.ru>
date:      Thu Aug 31 20:48:52 2017 +0300
description:
Version 0.1.13.

diffstat:

 CHANGES  |  13 +++++++++++++
 Makefile |   2 +-
 2 files changed, 14 insertions(+), 1 deletions(-)

diffs (30 lines):

diff -r 27aa477208f5 -r d548b78eb881 CHANGES
--- a/CHANGES	Thu Aug 31 20:27:31 2017 +0300
+++ b/CHANGES	Thu Aug 31 20:48:52 2017 +0300
@@ -1,3 +1,16 @@
+
+Changes with nJScript 0.1.13                                     31 Aug 2017
+
+    *) Feature: console.log() and console.help() methods in interactive
+       shell.
+
+    *) Feature: interactive shell prints backtrace on exception.
+
+    *) Feature: interactive shell by default if libedit is available.
+
+    *) Bugfix: processing of large files from stdin in command line mode.
+
+    *) Bugfix: improved editline detection.
 
 Changes with nJScript 0.1.12                                     08 Aug 2017
 
diff -r 27aa477208f5 -r d548b78eb881 Makefile
--- a/Makefile	Thu Aug 31 20:27:31 2017 +0300
+++ b/Makefile	Thu Aug 31 20:48:52 2017 +0300
@@ -1,5 +1,5 @@
 
-NJS_VER =	0.1.12
+NJS_VER =	0.1.13
 
 NXT_LIB =	nxt
 

From igor at sysoev.ru  Thu Aug 31 17:50:53 2017
From: igor at sysoev.ru (Igor Sysoev)
Date: Thu, 31 Aug 2017 17:50:53 +0000
Subject: [njs] Added tag 0.1.13 for changeset d548b78eb881
Message-ID: <hg.2a8aa9783fe9.1504201853.5965299922797593991@dev.nginx.com>

details:   http://hg.nginx.org/njs/rev/2a8aa9783fe9
branches:  
changeset: 405:2a8aa9783fe9
user:      Igor Sysoev <igor at sysoev.ru>
date:      Thu Aug 31 20:50:25 2017 +0300
description:
Added tag 0.1.13 for changeset d548b78eb881

diffstat:

 .hgtags |  1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diffs (8 lines):

diff -r d548b78eb881 -r 2a8aa9783fe9 .hgtags
--- a/.hgtags	Thu Aug 31 20:48:52 2017 +0300
+++ b/.hgtags	Thu Aug 31 20:50:25 2017 +0300
@@ -11,3 +11,4 @@ 5bd2833988222900f60ad9b330ebc44df3b30662
 b1456ef3e002376d9d146a8a02acf6a4a21748e9 0.1.10
 fc5df33f4e6b02a673daf3728ff690fb1e09b95e 0.1.11
 c07b060396be3622ca97b037a86076b61b850847 0.1.12
+d548b78eb881ca799aa6fc8ba459d076f7db5ac8 0.1.13