[nginx] Cache: added watermark to reduce IO load when keys_zone is full.

[Dmitry Volyntsev]

details:   http://hg.nginx.org/nginx/rev/c9d680b00744
branches:  
changeset: 6445:c9d680b00744
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Fri Mar 18 15:08:21 2016 +0300
description:
Cache: added watermark to reduce IO load when keys_zone is full.

When a keys_zone is full then each next request to the cache is
penalized.  That is, the cache has to evict older files to get a
slot from the keys_zone synchronously.  The patch introduces new
behavior in this scenario.  Manager will try to maintain available
free slots in the keys_zone by cleaning old files in the background.

diffstat:

 src/http/ngx_http_cache.h      |   2 ++
 src/http/ngx_http_file_cache.c |  39 +++++++++++++++++++++++++++++++++------
 2 files changed, 35 insertions(+), 6 deletions(-)

diffs (133 lines):

diff -r 043914d19be8 -r c9d680b00744 src/http/ngx_http_cache.h
--- a/src/http/ngx_http_cache.h	Fri Mar 18 14:27:30 2016 +0300
+++ b/src/http/ngx_http_cache.h	Fri Mar 18 15:08:21 2016 +0300
@@ -138,6 +138,8 @@ typedef struct {
     ngx_atomic_t                     cold;
     ngx_atomic_t                     loading;
     off_t                            size;
+    ngx_uint_t                       count;
+    ngx_uint_t                       watermark;
 } ngx_http_file_cache_sh_t;
 
 
diff -r 043914d19be8 -r c9d680b00744 src/http/ngx_http_file_cache.c
--- a/src/http/ngx_http_file_cache.c	Fri Mar 18 14:27:30 2016 +0300
+++ b/src/http/ngx_http_file_cache.c	Fri Mar 18 15:08:21 2016 +0300
@@ -62,6 +62,7 @@ static ngx_int_t ngx_http_file_cache_add
     ngx_http_cache_t *c);
 static ngx_int_t ngx_http_file_cache_delete_file(ngx_tree_ctx_t *ctx,
     ngx_str_t *path);
+static void ngx_http_file_cache_set_watermark(ngx_http_file_cache_t *cache);
 
 
 ngx_str_t  ngx_http_cache_status[] = {
@@ -147,6 +148,8 @@ ngx_http_file_cache_init(ngx_shm_zone_t 
     cache->sh->cold = 1;
     cache->sh->loading = 0;
     cache->sh->size = 0;
+    cache->sh->count = 0;
+    cache->sh->watermark = (ngx_uint_t) -1;
 
     cache->bsize = ngx_fs_bsize(cache->path->name.data);
 
@@ -861,6 +864,8 @@ ngx_http_file_cache_exists(ngx_http_file
     fcn = ngx_slab_calloc_locked(cache->shpool,
                                  sizeof(ngx_http_file_cache_node_t));
     if (fcn == NULL) {
+        ngx_http_file_cache_set_watermark(cache);
+
         ngx_shmtx_unlock(&cache->shpool->mutex);
 
         (void) ngx_http_file_cache_forced_expire(cache);
@@ -877,6 +882,8 @@ ngx_http_file_cache_exists(ngx_http_file
         }
     }
 
+    cache->sh->count++;
+
     ngx_memcpy((u_char *) &fcn->node.key, c->key, sizeof(ngx_rbtree_key_t));
 
     ngx_memcpy(fcn->key, &c->key[sizeof(ngx_rbtree_key_t)],
@@ -1631,6 +1638,7 @@ ngx_http_file_cache_free(ngx_http_cache_
         ngx_queue_remove(&fcn->queue);
         ngx_rbtree_delete(&cache->sh->rbtree, &fcn->node);
         ngx_slab_free_locked(cache->shpool, fcn);
+        cache->sh->count--;
         c->node = NULL;
     }
 
@@ -1883,6 +1891,7 @@ ngx_http_file_cache_delete(ngx_http_file
         ngx_queue_remove(q);
         ngx_rbtree_delete(&cache->sh->rbtree, &fcn->node);
         ngx_slab_free_locked(cache->shpool, fcn);
+        cache->sh->count--;
     }
 }
 
@@ -1892,8 +1901,9 @@ ngx_http_file_cache_manager(void *data)
 {
     ngx_http_file_cache_t  *cache = data;
 
-    off_t   size;
-    time_t  next, wait;
+    off_t       size;
+    time_t      next, wait;
+    ngx_uint_t  count, watermark;
 
     next = ngx_http_file_cache_expire(cache);
 
@@ -1904,13 +1914,16 @@ ngx_http_file_cache_manager(void *data)
         ngx_shmtx_lock(&cache->shpool->mutex);
 
         size = cache->sh->size;
+        count = cache->sh->count;
+        watermark = cache->sh->watermark;
 
         ngx_shmtx_unlock(&cache->shpool->mutex);
 
-        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
-                       "http file cache size: %O", size);
-
-        if (size < cache->max_size) {
+        ngx_log_debug3(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
+                       "http file cache size: %O c:%ui w:%i",
+                       size, count, (ngx_int_t) watermark);
+
+        if (size < cache->max_size && count < watermark) {
             return next;
         }
 
@@ -2094,12 +2107,16 @@ ngx_http_file_cache_add(ngx_http_file_ca
         fcn = ngx_slab_calloc_locked(cache->shpool,
                                      sizeof(ngx_http_file_cache_node_t));
         if (fcn == NULL) {
+            ngx_http_file_cache_set_watermark(cache);
+
             if (cache->fail_time != ngx_time()) {
                 cache->fail_time = ngx_time();
                 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
                            "could not allocate node%s", cache->shpool->log_ctx);
             }
 
+            cache->sh->count++;
+
             ngx_shmtx_unlock(&cache->shpool->mutex);
             return NGX_ERROR;
         }
@@ -2146,6 +2163,16 @@ ngx_http_file_cache_delete_file(ngx_tree
 }
 
 
+static void
+ngx_http_file_cache_set_watermark(ngx_http_file_cache_t *cache)
+{
+    cache->sh->watermark = cache->sh->count - cache->sh->count / 8;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
+                   "http file cache watermark: %ui", cache->sh->watermark);
+}
+
+
 time_t
 ngx_http_file_cache_valid(ngx_array_t *cache_valid, ngx_uint_t status)
 {