Antw: [PATCH 1 of 2] Access log: Support for disabling escaping

Johannes Baiter Johannes.Baiter at bsb-muenchen.de
Thu Feb 22 17:12:52 UTC 2018


Sorry, I accidentally submitted an incomplete version of the patch.
Here is the corrected version.

# HG changeset patch
# User Johannes Baiter <johannes.baiter at bsb-muenchen.de>
# Date 1519312196 -3600
#      Thu Feb 22 16:09:56 2018 +0100
# Node ID 6c1e7ddade7dd75677cb2821f346e862866ba4e3
# Parent  aa60f5799a4cbfc6887ef099c588e84a75c69785
Access log: support for disabling escaping.

diff -r aa60f5799a4c -r 6c1e7ddade7d
src/http/modules/ngx_http_log_module.c
--- a/src/http/modules/ngx_http_log_module.c	Thu Feb 22 12:42:29 2018
+0300
+++ b/src/http/modules/ngx_http_log_module.c	Thu Feb 22 16:09:56 2018
+0100
@@ -90,6 +90,13 @@
 } ngx_http_log_var_t;
 
 
+typedef enum {
+    ngx_http_log_escape_default = 0,
+    ngx_http_log_escape_json,
+    ngx_http_log_escape_none
+} ngx_http_log_escape_t;
+
+
 static void ngx_http_log_write(ngx_http_request_t *r, ngx_http_log_t
*log,
     u_char *buf, size_t len);
 static ssize_t ngx_http_log_script_write(ngx_http_request_t *r,
@@ -126,12 +133,16 @@
     ngx_http_log_op_t *op);
 
 static ngx_int_t ngx_http_log_variable_compile(ngx_conf_t *cf,
-    ngx_http_log_op_t *op, ngx_str_t *value, ngx_uint_t json);
+    ngx_http_log_op_t *op, ngx_str_t *value, ngx_http_log_escape_t
escape);
 static size_t ngx_http_log_variable_getlen(ngx_http_request_t *r,
     uintptr_t data);
 static u_char *ngx_http_log_variable(ngx_http_request_t *r, u_char
*buf,
     ngx_http_log_op_t *op);
 static uintptr_t ngx_http_log_escape(u_char *dst, u_char *src, size_t
size);
+static size_t
ngx_http_log_unescaped_variable_getlen(ngx_http_request_t *r,
+    uintptr_t data);
+static u_char *ngx_http_log_unescaped_variable(ngx_http_request_t *r,
u_char *buf,
+    ngx_http_log_op_t *op);
 static size_t ngx_http_log_json_variable_getlen(ngx_http_request_t
*r,
     uintptr_t data);
 static u_char *ngx_http_log_json_variable(ngx_http_request_t *r,
u_char *buf,
@@ -905,7 +916,7 @@
 
 static ngx_int_t
 ngx_http_log_variable_compile(ngx_conf_t *cf, ngx_http_log_op_t *op,
-    ngx_str_t *value, ngx_uint_t json)
+    ngx_str_t *value, ngx_http_log_escape_t escape)
 {
     ngx_int_t  index;
 
@@ -916,10 +927,14 @@
 
     op->len = 0;
 
-    if (json) {
+    if (escape == ngx_http_log_escape_json) {
         op->getlen = ngx_http_log_json_variable_getlen;
         op->run = ngx_http_log_json_variable;
 
+    } else if (escape == ngx_http_log_escape_none) {
+        op->getlen = ngx_http_log_unescaped_variable_getlen;
+        op->run = ngx_http_log_unescaped_variable;
+
     } else {
         op->getlen = ngx_http_log_variable_getlen;
         op->run = ngx_http_log_variable;
@@ -1033,6 +1048,38 @@
 
 
 static size_t
+ngx_http_log_unescaped_variable_getlen(ngx_http_request_t *r,
uintptr_t data)
+{
+    ngx_http_variable_value_t  *value;
+
+    value = ngx_http_get_indexed_variable(r, data);
+
+    if (value == NULL || value->not_found) {
+        return 0;
+    }
+
+    value->escape = 0;
+    return value->len;
+}
+
+
+static u_char *
+ngx_http_log_unescaped_variable(ngx_http_request_t *r, u_char *buf,
+                                ngx_http_log_op_t *op)
+{
+    ngx_http_variable_value_t  *value;
+
+    value = ngx_http_get_indexed_variable(r, op->data);
+
+    if (value == NULL || value->not_found) {
+        return buf;
+    }
+
+    return ngx_cpymem(buf, value->data, value->len);
+}
+
+
+static size_t
 ngx_http_log_json_variable_getlen(ngx_http_request_t *r, uintptr_t
data)
 {
     uintptr_t                   len;
@@ -1536,19 +1583,21 @@
     size_t               i, len;
     ngx_str_t           *value, var;
     ngx_int_t           *flush;
-    ngx_uint_t           bracket, json;
+    ngx_uint_t           bracket;
     ngx_http_log_op_t   *op;
     ngx_http_log_var_t  *v;
+    ngx_http_log_escape_t escape;
 
-    json = 0;
+    escape = ngx_http_log_escape_default;
     value = args->elts;
 
     if (s < args->nelts && ngx_strncmp(value[s].data, "escape=", 7) ==
0) {
         data = value[s].data + 7;
 
         if (ngx_strcmp(data, "json") == 0) {
-            json = 1;
-
+            escape = ngx_http_log_escape_json;
+        } else if (ngx_strcmp(data, "none") == 0) {
+            escape = ngx_http_log_escape_none;
         } else if (ngx_strcmp(data, "default") != 0) {
             ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
                                "unknown log format escaping \"%s\"",
data);
@@ -1636,7 +1685,7 @@
                     }
                 }
 
-                if (ngx_http_log_variable_compile(cf, op, &var, json)
+                if (ngx_http_log_variable_compile(cf, op, &var,
escape)
                     != NGX_OK)
                 {
                     return NGX_CONF_ERROR;
diff -r aa60f5799a4c -r 6c1e7ddade7d
src/stream/ngx_stream_log_module.c
--- a/src/stream/ngx_stream_log_module.c	Thu Feb 22 12:42:29 2018
+0300
+++ b/src/stream/ngx_stream_log_module.c	Thu Feb 22 16:09:56 2018
+0100
@@ -89,6 +89,13 @@
 } ngx_stream_log_var_t;
 
 
+typedef enum {
+    ngx_stream_log_escape_default = 0,
+    ngx_stream_log_escape_json,
+    ngx_stream_log_escape_none
+} ngx_stream_log_escape_t;
+
+
 static void ngx_stream_log_write(ngx_stream_session_t *s,
ngx_stream_log_t *log,
     u_char *buf, size_t len);
 static ssize_t ngx_stream_log_script_write(ngx_stream_session_t *s,
@@ -106,12 +113,16 @@
 static void ngx_stream_log_flush_handler(ngx_event_t *ev);
 
 static ngx_int_t ngx_stream_log_variable_compile(ngx_conf_t *cf,
-    ngx_stream_log_op_t *op, ngx_str_t *value, ngx_uint_t json);
+    ngx_stream_log_op_t *op, ngx_str_t *value, ngx_stream_log_escape_t
escape);
 static size_t ngx_stream_log_variable_getlen(ngx_stream_session_t *s,
     uintptr_t data);
 static u_char *ngx_stream_log_variable(ngx_stream_session_t *s, u_char
*buf,
     ngx_stream_log_op_t *op);
 static uintptr_t ngx_stream_log_escape(u_char *dst, u_char *src,
size_t size);
+static size_t
ngx_stream_log_unescaped_variable_getlen(ngx_stream_session_t *s,
+    uintptr_t data);
+static u_char *ngx_stream_log_unescaped_variable(ngx_stream_session_t
*s,
+    u_char *buf, ngx_stream_log_op_t *op);
 static size_t ngx_stream_log_json_variable_getlen(ngx_stream_session_t
*s,
     uintptr_t data);
 static u_char *ngx_stream_log_json_variable(ngx_stream_session_t *s,
@@ -682,7 +693,7 @@
 
 static ngx_int_t
 ngx_stream_log_variable_compile(ngx_conf_t *cf, ngx_stream_log_op_t
*op,
-    ngx_str_t *value, ngx_uint_t json)
+    ngx_str_t *value, ngx_stream_log_escape_t escape)
 {
     ngx_int_t  index;
 
@@ -693,10 +704,14 @@
 
     op->len = 0;
 
-    if (json) {
+    if (escape == ngx_stream_log_escape_json) {
         op->getlen = ngx_stream_log_json_variable_getlen;
         op->run = ngx_stream_log_json_variable;
 
+    } else if (escape == ngx_stream_log_escape_none) {
+        op->getlen = ngx_stream_log_unescaped_variable_getlen;
+        op->run = ngx_stream_log_unescaped_variable;
+
     } else {
         op->getlen = ngx_stream_log_variable_getlen;
         op->run = ngx_stream_log_variable;
@@ -811,6 +826,38 @@
 
 
 static size_t
+ngx_stream_log_unescaped_variable_getlen(ngx_stream_session_t *s,
uintptr_t data)
+{
+    ngx_stream_variable_value_t  *value;
+
+    value = ngx_stream_get_indexed_variable(s, data);
+
+    if (value == NULL || value->not_found) {
+        return 0;
+    }
+
+    value->escape = 0;
+    return value->len;
+}
+
+
+static u_char *
+ngx_stream_log_unescaped_variable(ngx_stream_session_t *s, u_char
*buf,
+                                  ngx_stream_log_op_t *op)
+{
+    ngx_stream_variable_value_t  *value;
+
+    value = ngx_stream_get_indexed_variable(s, op->data);
+
+    if (value == NULL || value->not_found) {
+        return buf;
+    }
+
+    return ngx_cpymem(buf, value->data, value->len);
+}
+
+
+static size_t
 ngx_stream_log_json_variable_getlen(ngx_stream_session_t *s, uintptr_t
data)
 {
     uintptr_t                     len;
@@ -1265,17 +1312,21 @@
     size_t                 i, len;
     ngx_str_t             *value, var;
     ngx_int_t             *flush;
-    ngx_uint_t             bracket, json;
+    ngx_uint_t             bracket;
     ngx_stream_log_op_t   *op;
+    ngx_stream_log_escape_t escape;
 
-    json = 0;
+    escape = ngx_stream_log_escape_default;
     value = args->elts;
 
     if (s < args->nelts && ngx_strncmp(value[s].data, "escape=", 7) ==
0) {
         data = value[s].data + 7;
 
         if (ngx_strcmp(data, "json") == 0) {
-            json = 1;
+            escape = ngx_stream_log_escape_json;
+
+        } else if (ngx_strcmp(data, "none") == 0) {
+            escape = ngx_stream_log_escape_none;
 
         } else if (ngx_strcmp(data, "default") != 0) {
             ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
@@ -1350,7 +1401,7 @@
                     goto invalid;
                 }
 
-                if (ngx_stream_log_variable_compile(cf, op, &var,
json)
+                if (ngx_stream_log_variable_compile(cf, op, &var,
escape)
                     != NGX_OK)
                 {
                     return NGX_CONF_ERROR; 
 
>>> "Johannes Baiter" <Johannes.Baiter at bsb-muenchen.de> 2/22/2018 5:08
PM >>> 
This patch adds a "none" option for the "escape" parameter on the
"log_format" directive.

Selecting this option will perform no escaping of variables in the
access log output and write the variable to the file as-is.
The use case for this is more freedom for custom logging setups.

In our case, we use structured logging in JSON. For this, we build a
JSON string in a Lua script that is then stored in a nginx variable.
The log_format in this case is simply the variable:

  # The script sets the $json_log variable
  log_by_lua_file /etc/nginx/lua/scripts/json_logging.lua;
  log_format json $json_log;

  access_log /local/nginx/log/access.log json;
  error_log /local/nginx/log/error.log;

Both "default" and "json" do not work in this case, since either will
result in invalid JSON output.
Updated test suites for both ngx_http and ngx_stream are provided in
the second patch.


# HG changeset patch
# User Johannes Baiter <johannes.baiter at bsb-muenchen.de>
# Date 1519312196 -3600
#      Thu Feb 22 16:09:56 2018 +0100
# Node ID 267e08661efcb58a9b4cd5fd941486b88a37f4f4
# Parent  aa60f5799a4cbfc6887ef099c588e84a75c69785
Access log: support for disabling escaping.

diff -r aa60f5799a4c -r 267e08661efc
src/http/modules/ngx_http_log_module.c
--- a/src/http/modules/ngx_http_log_module.c	Thu Feb 22 12:42:29
2018
+0300
+++ b/src/http/modules/ngx_http_log_module.c	Thu Feb 22 16:09:56
2018
+0100
@@ -90,6 +90,13 @@
 } ngx_http_log_var_t;
 
 
+typedef enum {
+    ngx_http_log_escape_default = 0,
+    ngx_http_log_escape_json,
+    ngx_http_log_escape_none
+} ngx_http_log_escape_t;
+
+
 static void ngx_http_log_write(ngx_http_request_t *r, ngx_http_log_t
*log,
     u_char *buf, size_t len);
 static ssize_t ngx_http_log_script_write(ngx_http_request_t *r,
@@ -126,12 +133,16 @@
     ngx_http_log_op_t *op);
 
 static ngx_int_t ngx_http_log_variable_compile(ngx_conf_t *cf,
-    ngx_http_log_op_t *op, ngx_str_t *value, ngx_uint_t json);
+    ngx_http_log_op_t *op, ngx_str_t *value, ngx_http_log_escape_t
escape);
 static size_t ngx_http_log_variable_getlen(ngx_http_request_t *r,
     uintptr_t data);
 static u_char *ngx_http_log_variable(ngx_http_request_t *r, u_char
*buf,
     ngx_http_log_op_t *op);
 static uintptr_t ngx_http_log_escape(u_char *dst, u_char *src, size_t
size);
+static size_t
ngx_http_log_unescaped_variable_getlen(ngx_http_request_t *r,
+    uintptr_t data);
+static u_char *ngx_http_log_unescaped_variable(ngx_http_request_t *r,
u_char *buf,
+    ngx_http_log_op_t *op);
 static size_t ngx_http_log_json_variable_getlen(ngx_http_request_t
*r,
     uintptr_t data);
 static u_char *ngx_http_log_json_variable(ngx_http_request_t *r,
u_char *buf,
@@ -905,7 +916,7 @@
 
 static ngx_int_t
 ngx_http_log_variable_compile(ngx_conf_t *cf, ngx_http_log_op_t *op,
-    ngx_str_t *value, ngx_uint_t json)
+    ngx_str_t *value, ngx_http_log_escape_t escape)
 {
     ngx_int_t  index;
 
@@ -916,10 +927,12 @@
 
     op->len = 0;
 
-    if (json) {
+    if (escape == ngx_http_log_escape_json) {
         op->getlen = ngx_http_log_json_variable_getlen;
         op->run = ngx_http_log_json_variable;
-
+    } else if (escape == ngx_http_log_escape_none) {
+        op->getlen = ngx_http_log_unescaped_variable_getlen;
+        op->run = ngx_http_log_unescaped_variable;
     } else {
         op->getlen = ngx_http_log_variable_getlen;
         op->run = ngx_http_log_variable;
@@ -1033,6 +1046,38 @@
 
 
 static size_t
+ngx_http_log_unescaped_variable_getlen(ngx_http_request_t *r,
uintptr_t data)
+{
+    ngx_http_variable_value_t  *value;
+
+    value = ngx_http_get_indexed_variable(r, data);
+
+    if (value == NULL || value->not_found) {
+        return 0;
+    }
+
+    value->escape = 0;
+    return value->len;
+}
+
+
+static u_char *
+ngx_http_log_unescaped_variable(ngx_http_request_t *r, u_char *buf,
+                                ngx_http_log_op_t *op)
+{
+    ngx_http_variable_value_t  *value;
+
+    value = ngx_http_get_indexed_variable(r, op->data);
+
+    if (value == NULL || value->not_found) {
+        return buf;
+    }
+
+    return ngx_cpymem(buf, value->data, value->len);
+}
+
+
+static size_t
 ngx_http_log_json_variable_getlen(ngx_http_request_t *r, uintptr_t
data)
 {
     uintptr_t                   len;
@@ -1536,19 +1581,21 @@
     size_t               i, len;
     ngx_str_t           *value, var;
     ngx_int_t           *flush;
-    ngx_uint_t           bracket, json;
+    ngx_uint_t           bracket;
     ngx_http_log_op_t   *op;
     ngx_http_log_var_t  *v;
+    ngx_http_log_escape_t escape;
 
-    json = 0;
+    escape = ngx_http_log_escape_default;
     value = args->elts;
 
     if (s < args->nelts && ngx_strncmp(value[s].data, "escape=", 7)
==
0) {
         data = value[s].data + 7;
 
         if (ngx_strcmp(data, "json") == 0) {
-            json = 1;
-
+            escape = ngx_http_log_escape_json;
+        } else if (ngx_strcmp(data, "none") == 0) {
+            escape = ngx_http_log_escape_none;
         } else if (ngx_strcmp(data, "default") != 0) {
             ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
                                "unknown log format escaping \"%s\"",
data);
@@ -1636,7 +1683,7 @@
                     }
                 }
 
-                if (ngx_http_log_variable_compile(cf, op, &var, json)
+                if (ngx_http_log_variable_compile(cf, op, &var,
escape)
                     != NGX_OK)
                 {
                     return NGX_CONF_ERROR;
diff -r aa60f5799a4c -r 267e08661efc
src/stream/ngx_stream_log_module.c
--- a/src/stream/ngx_stream_log_module.c	Thu Feb 22 12:42:29
2018
+0300
+++ b/src/stream/ngx_stream_log_module.c	Thu Feb 22 16:09:56
2018
+0100
@@ -89,6 +89,13 @@
 } ngx_stream_log_var_t;
 
 
+typedef enum {
+    ngx_stream_log_escape_default = 0,
+    ngx_stream_log_escape_json,
+    ngx_stream_log_escape_none
+} ngx_stream_log_escape_t;
+
+
 static void ngx_stream_log_write(ngx_stream_session_t *s,
ngx_stream_log_t *log,
     u_char *buf, size_t len);
 static ssize_t ngx_stream_log_script_write(ngx_stream_session_t *s,
@@ -106,7 +113,7 @@
 static void ngx_stream_log_flush_handler(ngx_event_t *ev);
 
 static ngx_int_t ngx_stream_log_variable_compile(ngx_conf_t *cf,
-    ngx_stream_log_op_t *op, ngx_str_t *value, ngx_uint_t json);
+    ngx_stream_log_op_t *op, ngx_str_t *value,
ngx_stream_log_escape_t
escape);
 static size_t ngx_stream_log_variable_getlen(ngx_stream_session_t *s,
     uintptr_t data);
 static u_char *ngx_stream_log_variable(ngx_stream_session_t *s,
u_char
*buf,
@@ -682,7 +689,7 @@
 
 static ngx_int_t
 ngx_stream_log_variable_compile(ngx_conf_t *cf, ngx_stream_log_op_t
*op,
-    ngx_str_t *value, ngx_uint_t json)
+    ngx_str_t *value, ngx_stream_log_escape_t escape)
 {
     ngx_int_t  index;
 
@@ -693,10 +700,14 @@
 
     op->len = 0;
 
-    if (json) {
+    if (escape == ngx_stream_log_escape_json) {
         op->getlen = ngx_stream_log_json_variable_getlen;
         op->run = ngx_stream_log_json_variable;
 
+    } else if (escape == ngx_stream_log_escape_none) {
+        op->getlen = ngx_stream_log_unescaped_variable_getlen;
+        op->run = ngx_stream_log_unescaped_variable;
+
     } else {
         op->getlen = ngx_stream_log_variable_getlen;
         op->run = ngx_stream_log_variable;
@@ -811,6 +822,38 @@
 
 
 static size_t
+ngx_stream_log_unescaped_variable_getlen(ngx_stream_session_t *s,
uintptr_t data)
+{
+    ngx_stream_variable_value_t  *value;
+
+    value = ngx_stream_get_indexed_variable(s, data);
+
+    if (value == NULL || value->not_found) {
+        return 0;
+    }
+
+    value->escape = 0;
+    return value->len;
+}
+
+
+static u_char *
+ngx_http_log_unescaped_variable(ngx_stream_session_t *s, u_char *buf,
+                                ngx_stream_log_op_t *op)
+{
+    ngx_stream_variable_value_t  *value;
+
+    value = ngx_stream_get_indexed_variable(s, op->data);
+
+    if (value == NULL || value->not_found) {
+        return buf;
+    }
+
+    return ngx_cpymem(buf, value->data, value->len);
+}
+
+
+static size_t
 ngx_stream_log_json_variable_getlen(ngx_stream_session_t *s,
uintptr_t
data)
 {
     uintptr_t                     len;
@@ -1265,17 +1308,21 @@
     size_t                 i, len;
     ngx_str_t             *value, var;
     ngx_int_t             *flush;
-    ngx_uint_t             bracket, json;
+    ngx_uint_t             bracket;
     ngx_stream_log_op_t   *op;
+    ngx_stream_log_escape_t escape;
 
-    json = 0;
+    escape = ngx_stream_log_escape_default;
     value = args->elts;
 
     if (s < args->nelts && ngx_strncmp(value[s].data, "escape=", 7)
==
0) {
         data = value[s].data + 7;
 
         if (ngx_strcmp(data, "json") == 0) {
-            json = 1;
+            escape = ngx_stream_log_escape_json;
+
+        } else if (ngx_strcmp(data, "none") == 0) {
+            escape = ngx_stream_log_escape_none;
 
         } else if (ngx_strcmp(data, "default") != 0) {
             ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
@@ -1350,7 +1397,7 @@
                     goto invalid;
                 }
 
-                if (ngx_stream_log_variable_compile(cf, op, &var,
json)
+                if (ngx_stream_log_variable_compile(cf, op, &var,
escape)
                     != NGX_OK)
                 {
                     return NGX_CONF_ERROR;

_______________________________________________
nginx-devel mailing list
nginx-devel at nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel



More information about the nginx-devel mailing list