[PATCH 01 of 31] Reuse keepalive connections if worker connections aren't enough

[Maxim Dounin]

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1297774382 -10800
# Node ID e498d1353a561a8e427640ff76bbde298e3ee5d3
# Parent  f02ddde4a240628b002b3fc85aa19bb38e7c5d85
Reuse keepalive connections if worker connections aren't enough.

This allows to set keepalive_timeout high enough, and don't care if you
have enough worker_connections to handle load spikes (either natural, or
caused by CARP failover or something like).  LRU queue will effectively
lower keepalive timeout if there are not enough worker connections.

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -12,6 +12,9 @@
 ngx_os_io_t  ngx_io;
 
 
+static void ngx_drain_connections(void);
+
+
 ngx_listening_t *
 ngx_create_listening(ngx_conf_t *cf, void *sockaddr, socklen_t socklen)
 {
@@ -719,6 +722,11 @@ ngx_get_connection(ngx_socket_t s, ngx_l
     c = ngx_cycle->free_connections;
 
     if (c == NULL) {
+        ngx_drain_connections();
+        c = ngx_cycle->free_connections;
+    }
+
+    if (c == NULL) {
         ngx_log_error(NGX_LOG_ALERT, log, 0,
                       "%ui worker_connections are not enough",
                       ngx_cycle->connection_n);
@@ -861,6 +869,8 @@ ngx_close_connection(ngx_connection_t *c
 
 #endif
 
+    ngx_reusable_connection(c, 0);
+
     log_error = c->log_error;
 
     ngx_free_connection(c);
@@ -900,6 +910,51 @@ ngx_close_connection(ngx_connection_t *c
 }
 
 
+void
+ngx_reusable_connection(ngx_connection_t *c, ngx_uint_t reusable)
+{
+    ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
+                   "reusable connection: %ui", reusable);
+
+    if (c->reusable) {
+        ngx_queue_remove(&c->queue);
+    }
+
+    c->reusable = reusable;
+
+    if (reusable) {
+        /* need cast as ngx_cycle is volatile */
+
+        ngx_queue_insert_head(
+            (ngx_queue_t *) &ngx_cycle->reusable_connections_queue, &c->queue);
+    }
+}
+
+
+static void
+ngx_drain_connections(void)
+{
+    ngx_int_t          i;
+    ngx_queue_t       *q;
+    ngx_connection_t  *c;
+
+    for (i = 0; i < 32; i++) {
+        if (ngx_queue_empty(&ngx_cycle->reusable_connections_queue)) {
+            break;
+        }
+
+        q = ngx_queue_last(&ngx_cycle->reusable_connections_queue);
+        c = ngx_queue_data(q, ngx_connection_t, queue);
+
+        ngx_log_debug0(NGX_LOG_DEBUG_CORE, c->log, 0,
+                       "reusing connection");
+
+        c->close = 1;
+        c->read->handler(c->read);
+    }
+}
+
+
 ngx_int_t
 ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
     ngx_uint_t port)
diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -135,6 +135,8 @@ struct ngx_connection_s {
 
     ngx_buf_t          *buffer;
 
+    ngx_queue_t         queue;
+
     ngx_atomic_uint_t   number;
 
     ngx_uint_t          requests;
@@ -150,6 +152,7 @@ struct ngx_connection_s {
     unsigned            destroyed:1;
 
     unsigned            idle:1;
+    unsigned            reusable:1;
     unsigned            close:1;
 
     unsigned            sendfile:1;
@@ -186,5 +189,6 @@ ngx_int_t ngx_connection_error(ngx_conne
 ngx_connection_t *ngx_get_connection(ngx_socket_t s, ngx_log_t *log);
 void ngx_free_connection(ngx_connection_t *c);
 
+void ngx_reusable_connection(ngx_connection_t *c, ngx_uint_t reusable);
 
 #endif /* _NGX_CONNECTION_H_INCLUDED_ */
diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c
--- a/src/core/ngx_cycle.c
+++ b/src/core/ngx_cycle.c
@@ -181,6 +181,9 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
     cycle->listening.pool = pool;
 
 
+    ngx_queue_init(&cycle->reusable_connections_queue);
+
+
     cycle->conf_ctx = ngx_pcalloc(pool, ngx_max_module * sizeof(void *));
     if (cycle->conf_ctx == NULL) {
         ngx_destroy_pool(pool);
diff --git a/src/core/ngx_cycle.h b/src/core/ngx_cycle.h
--- a/src/core/ngx_cycle.h
+++ b/src/core/ngx_cycle.h
@@ -44,6 +44,8 @@ struct ngx_cycle_s {
     ngx_connection_t         *free_connections;
     ngx_uint_t                free_connection_n;
 
+    ngx_queue_t               reusable_connections_queue;
+
     ngx_array_t               listening;
     ngx_array_t               pathes;
     ngx_list_t                open_files;
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -2589,6 +2589,7 @@ ngx_http_set_keepalive(ngx_http_request_
 #endif
 
     c->idle = 1;
+    ngx_reusable_connection(c, 1);
 
     if (rev->ready) {
         ngx_post_event(rev, &ngx_posted_events);
@@ -2698,6 +2699,7 @@ ngx_http_keepalive_handler(ngx_event_t *
     c->log->action = "reading client request line";
 
     c->idle = 0;
+    ngx_reusable_connection(c, 0);
 
     ngx_http_init_request(rev);
 }