[nginx] Cache: ignore long locked entries during forced expire.

[Dmitry Volyntsev]

details:   http://hg.nginx.org/nginx/rev/ab199f0eb8e8
branches:  
changeset: 7002:ab199f0eb8e8
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Thu May 18 18:39:16 2017 +0300
description:
Cache: ignore long locked entries during forced expire.

Abnormally exited workers may leave locked cache entries, this can
result in the cache size on disk exceeding max_size and shared memory
exhaustion.

This change mitigates the issue by ignoring locked entries during forced
expire.  It also increases the visibility of the problem by logging such
entries.

diffstat:

 src/http/ngx_http_file_cache.c |  57 ++++++++++++++++++++++++++++++++---------
 1 files changed, 44 insertions(+), 13 deletions(-)

diffs (91 lines):

diff -r 08537eab4f23 -r ab199f0eb8e8 src/http/ngx_http_file_cache.c
--- a/src/http/ngx_http_file_cache.c	Thu May 18 14:17:00 2017 +0300
+++ b/src/http/ngx_http_file_cache.c	Thu May 18 18:39:16 2017 +0300
@@ -1700,13 +1700,14 @@ ngx_http_file_cache_cleanup(void *data)
 static time_t
 ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache)
 {
-    u_char                      *name;
+    u_char                      *name, *p;
     size_t                       len;
     time_t                       wait;
     ngx_uint_t                   tries;
     ngx_path_t                  *path;
-    ngx_queue_t                 *q;
+    ngx_queue_t                 *q, *sentinel;
     ngx_http_file_cache_node_t  *fcn;
+    u_char                       key[2 * NGX_HTTP_CACHE_KEY_LEN];
 
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
                    "http file cache forced expire");
@@ -1723,13 +1724,21 @@ ngx_http_file_cache_forced_expire(ngx_ht
 
     wait = 10;
     tries = 20;
+    sentinel = NULL;
 
     ngx_shmtx_lock(&cache->shpool->mutex);
 
-    for (q = ngx_queue_last(&cache->sh->queue);
-         q != ngx_queue_sentinel(&cache->sh->queue);
-         q = ngx_queue_prev(q))
-    {
+    for ( ;; ) {
+        if (ngx_queue_empty(&cache->sh->queue)) {
+            break;
+        }
+
+        q = ngx_queue_last(&cache->sh->queue);
+
+        if (q == sentinel) {
+            break;
+        }
+
         fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
 
         ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
@@ -1740,15 +1749,37 @@ ngx_http_file_cache_forced_expire(ngx_ht
         if (fcn->count == 0) {
             ngx_http_file_cache_delete(cache, q, name);
             wait = 0;
-
-        } else {
-            if (--tries) {
-                continue;
-            }
-
-            wait = 1;
+            break;
         }
 
+        p = ngx_hex_dump(key, (u_char *) &fcn->node.key,
+                         sizeof(ngx_rbtree_key_t));
+        len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t);
+        (void) ngx_hex_dump(p, fcn->key, len);
+
+        /*
+         * abnormally exited workers may leave locked cache entries,
+         * and although it may be safe to remove them completely,
+         * we prefer to just move them to the top of the inactive queue
+         */
+
+        ngx_queue_remove(q);
+        fcn->expire = ngx_time() + cache->inactive;
+        ngx_queue_insert_head(&cache->sh->queue, &fcn->queue);
+
+        ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
+                      "ignore long locked inactive cache entry %*s, count:%d",
+                      (size_t) 2 * NGX_HTTP_CACHE_KEY_LEN, key, fcn->count);
+
+        if (sentinel == NULL) {
+            sentinel = q;
+        }
+
+        if (--tries) {
+            continue;
+        }
+
+        wait = 1;
         break;
     }