[PATCH] Proxy: support configuration of socket buffer sizes

[Karstens, Nate]

# HG changeset patch
# User Nate Karstens <nate.karstens at garmin.com>
# Date 1493467011 18000
#      Sat Apr 29 06:56:51 2017 -0500
# Node ID 1251a543804b17941b2c96b84bd1f4e58a37bc15
# Parent  8801ff7d58e1650c9d1abb50e09f5979e4f9ffbf
Proxy: support configuration of socket buffer sizes

Allows the size of the buffers used by the TCP sockets
for HTTP proxy connections to be configured. The new
configuration directives are:

 * proxy_socket_rcvbuf
 * proxy_socket_sndbuf

These correspond with the SO_RCVBUF and SO_SNDBUF socket
options, respectively.

This is be useful in cases where the proxy processes
received data slowly. Data was being buffered in three
separate TCP buffers (nginx-from-client receive, nginx-
to-proxy send, and proxy-from-nginx receive). The
cumulative effect is that the client thinks it has
sent all of the data, but times out waiting for a reply
from the proxy, which cannot reply because it is still
processing the data in its buffers.

Signed-off-by: Nate Karstens <nate.karstens at garmin.com>

diff -r 8801ff7d58e1 -r 1251a543804b src/event/ngx_event_connect.c
--- a/src/event/ngx_event_connect.c     Tue Apr 25 23:39:13 2017 +0300
+++ b/src/event/ngx_event_connect.c     Sat Apr 29 06:56:51 2017 -0500
@@ -73,6 +73,16 @@
         }
     }

+    if (pc->sndbuf) {
+        if (setsockopt(s, SOL_SOCKET, SO_SNDBUF,
+                       (const void *) &pc->sndbuf, sizeof(int)) == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                          "setsockopt(SO_SNDBUF) failed");
+            goto failed;
+        }
+    }
+
     if (ngx_nonblocking(s) == -1) {
         ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
                       ngx_nonblocking_n " failed");
diff -r 8801ff7d58e1 -r 1251a543804b src/event/ngx_event_connect.h
--- a/src/event/ngx_event_connect.h     Tue Apr 25 23:39:13 2017 +0300
+++ b/src/event/ngx_event_connect.h     Sat Apr 29 06:56:51 2017 -0500
@@ -57,6 +57,7 @@

     int                              type;
     int                              rcvbuf;
+    int                              sndbuf;

     ngx_log_t                       *log;

diff -r 8801ff7d58e1 -r 1251a543804b src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c  Tue Apr 25 23:39:13 2017 +0300
+++ b/src/http/modules/ngx_http_proxy_module.c  Sat Apr 29 06:56:51 2017 -0500
@@ -90,6 +90,9 @@
     ngx_uint_t                     headers_hash_max_size;
     ngx_uint_t                     headers_hash_bucket_size;

+    ngx_int_t                      rcvbuf;
+    ngx_int_t                      sndbuf;
+
 #if (NGX_HTTP_SSL)
     ngx_uint_t                     ssl;
     ngx_uint_t                     ssl_protocols;
@@ -629,6 +632,20 @@
       offsetof(ngx_http_proxy_loc_conf_t, http_version),
       &ngx_http_proxy_http_version },

+    { ngx_string("proxy_socket_rcvbuf"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_proxy_loc_conf_t, rcvbuf),
+      NULL },
+
+    { ngx_string("proxy_socket_sndbuf"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_proxy_loc_conf_t, sndbuf),
+      NULL },
+
 #if (NGX_HTTP_SSL)

     { ngx_string("proxy_ssl_session_reuse"),
@@ -905,6 +922,13 @@

     u->buffering = plcf->upstream.buffering;

+    if (plcf->rcvbuf != NGX_CONF_UNSET) {
+        u->peer.rcvbuf = plcf->rcvbuf;
+    }
+    if (plcf->sndbuf != NGX_CONF_UNSET) {
+        u->peer.sndbuf = plcf->sndbuf;
+    }
+
     u->pipe = ngx_pcalloc(r->pool, sizeof(ngx_event_pipe_t));
     if (u->pipe == NULL) {
         return NGX_HTTP_INTERNAL_SERVER_ERROR;
@@ -2902,6 +2926,9 @@
     conf->headers_hash_max_size = NGX_CONF_UNSET_UINT;
     conf->headers_hash_bucket_size = NGX_CONF_UNSET_UINT;

+    conf->rcvbuf = NGX_CONF_UNSET;
+    conf->sndbuf = NGX_CONF_UNSET;
+
     ngx_str_set(&conf->upstream.module, "proxy");

     return conf;
@@ -3297,6 +3324,12 @@
     ngx_conf_merge_uint_value(conf->headers_hash_bucket_size,
                               prev->headers_hash_bucket_size, 64);

+    ngx_conf_merge_value(conf->rcvbuf,
+                         prev->rcvbuf, NGX_CONF_UNSET);
+
+    ngx_conf_merge_value(conf->sndbuf,
+                         prev->sndbuf, NGX_CONF_UNSET);
+
     conf->headers_hash_bucket_size = ngx_align(conf->headers_hash_bucket_size,
                                                ngx_cacheline_size);


________________________________

CONFIDENTIALITY NOTICE: This email and any attachments are for the sole use of the intended recipient(s) and contain information that may be Garmin confidential and/or Garmin legally privileged. If you have received this email in error, please notify the sender by reply email and delete the message. Any disclosure, copying, distribution or use of this communication (including attachments) by someone other than the intended recipient is prohibited. Thank you.