ngx_slab_alloc() failed: no memory in cache keys zone

[Nick Urbanik]

Dear Folks,

On 19/09/17 10:46 +1000, Nick Urbanik wrote:
>We have this message repeatedly, despite increasing keys_zone size by
>a factor of three to:
>
>proxy_cache_path            /srv/mycache levels=1:2 keys_zone=myzone:150m inactive=15d;
>
>This caused the errors to stop for three or four hours and then back
>again.
>
>The cache itself is quite big:
>sudo du -s /srv/mycache
>1826303379	/srv/mycache
>
>and we have many messages like this in our logs:
>unlink() "/srv/mycache/c/fc/4e8755ecadf5a82ec7208c16d8ddbfcc" failed (2: No such file or directory)
>
>Any suggestions most welcome.

I omitted important details:
$ rpm -q nginx
nginx-1.6.2-1.el6.ngx.x86_64

nginx.conf:
$ sanitise-nginx /etc/nginx/nginx.conf 
user  nginx;
worker_processes  8;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;

error_log  /var/log/nginx/error.log;

pid        /var/run/nginx.pid;

events {
    worker_connections  4096;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format   main  '$remote_addr - $remote_user [$time_local] "$request" '
                       '$status $request_time $body_bytes_sent "$http_referer" '
                       '"$http_user_agent" "$http_x_forwarded_for" $sent_http_content_type '
                       'CACHE KEY(s/ph/u) => $scheme $proxy_host $uri -> REQUEST_URI: $request_uri '
                       'UPSTREAM => $upstream_cache_status $upstream_status $upstream_addr';    

    access_log /var/log/nginx/access.log main;

    sendfile        on;

    keepalive_timeout  65;

    proxy_buffering             on;                             # default
    proxy_cache_path            /srv/mycache levels=1:2 keys_zone=mycache:150m inactive=15d;
    proxy_temp_path             /srv/mycache_tmp/my.server;
    proxy_cache_key             $scheme$proxy_host$uri;
    proxy_max_temp_file_size    4096m;
    proxy_connect_timeout       5s;                             # how long before upstream dead
    proxy_next_upstream         error timeout invalid_header;   # default
    proxy_buffers               10 1M;

    upstream mycache_origin {
        server my.origin.com:80 max_fails=1 fail_timeout=180;
    }
    
    include /etc/nginx/conf.d/acl.conf;
    include /etc/nginx/conf.d/default.conf;
}

I reduced the time
proxy_cache_valid 200 90d
by half from 180d:

$ sanitise-nginx /etc/nginx/conf.d/default.conf 
server {
    listen       80 default_server;
    server_name  my.server.name.com;

    location /server-status {
        access_log off;
        stub_status on;
    }

    set $range $http_range;
    if ($range = "bytes=0-") {
        set $range "";
    }

    location / {
        proxy_cache             mycache;    # cache keys db
        proxy_cache_valid       200 90d;    # http 200
        proxy_cache_valid       404 1m;     # http 404
        proxy_cache_valid       any 10m;    # any other

        proxy_ignore_headers    "Cache-Control";
        proxy_ignore_headers    "Expires";

        proxy_set_header        Host               my.origin.com;
        proxy_set_header        X-Real-IP          $remote_addr;
        proxy_set_header        X-Forwarded-For    $proxy_add_x_forwarded_for;
        proxy_set_header        Upstream           $upstream_cache_status;
        proxy_set_header        Range              $range;

        proxy_no_cache          0;          # do cache
        proxy_cache_bypass      0;          # take from cache first
        proxy_pass              http://mycache_origin;
    }

    error_page  404              /404.html;
    location = /404.html {
        access_log off;
        root   /usr/share/nginx/html;
    }

    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        access_log off;
        root   /usr/share/nginx/html;
    }
}

If any other details would help, please let me know.
-- 
Nick Urbanik http://nicku.org 808-71011 nick.urbanik at optusnet.com.au
GPG: 7FFA CDC7 5A77 0558 DC7A 790A 16DF EC5B BB9D 2C24  ID: BB9D2C24
I disclaim, therefore I am.