Proxy-cache serving expired content

[frank_r]

Hello list,

we are using nginx as proxy/loadbalancer and cache in front of our
serverpark consisting of 8 apache/php-servers.
We are serving an average of 200 requests per second.

Performance and flexibility is second to none, however we are facing a
problem which we really need to solve:

We are experiencing a bug which should have been fixed in 0.8.14:
> *) Bugfix: an expired cached response might stick in the "UPDATING" 
state.

Symptoms are: Files way beyond their expire time are still being served,
the logs state "UPDATING". 
Restarting nginx and therefore triggering the "cache loader" process
leads to a cleanup of the stale file. 
This workaround however is no option for production since we have cache
times of as little as 1 min and we obviously can not restart nginx every
minute. 

I know there has been some work done in ngx_http_file_cache.c
(c->node->updating, etc.) however there still seems to be some race. I
would be more than willing to help given some pointers....

Since I have not found anything about this issue somewhere in the web
recently and we seem to be the only ones experiencing this, I meanwhile
even tend to think it is caused by our configuration (attached).

Any Help much appreciated, since we really want to stick with nginx (our
customer does not in the meantime).

Thanks,
Frank

---------
Config (somewhat shortened and anonymised):
---------
worker_processes  4;
worker_rlimit_nofile 10240;
working_directory /var/lib/nginx/cores/;
worker_rlimit_core 500M;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
        worker_connections  8192;
}
http {
        server_tokens off;
        include    mime.types;
        default_type  application/octet-stream;
        log_format main '$remote_addr - $remote_user [$time_local]
"$request" '
                                '$status $body_bytes_sent
"$http_referer" '
                                '"$http_user_agent"
"$http_x_forwarded_for"';
        access_log /var/log/nginx/access.log main;
        sendfile on;
        keepalive_timeout 65;
        gzip on;
        proxy_cache_path /var/ramdisk/nginx/abc levels=1:2:1
                        keys_zone=abc:1022m
                        inactive=60m  max_size=5500m;
        proxy_cache_valid 200   45m;
        proxy_cache_valid 302 301 303 10m;
        proxy_cache_valid 404   10m;
        proxy_cache_valid any   10m;
        proxy_cache_valid 401   1s;
        proxy_cache_use_stale updating error timeout invalid_header;
        proxy_cache_key $scheme$host$uri$is_args$args;
        proxy_temp_path /var/cache/nginx/tmp_nginx; 
        upstream abc_pool {
                server 62.146.96.41 weight=3;
                server 62.146.96.44 weight=3;
                server 62.146.96.45 weight=3;
                server 62.146.96.46 weight=3;
                server 62.146.96.47 weight=3;
                server 62.146.96.48 weight=1;
                server 62.146.96.49 weight=3;
                server 62.146.96.53 weight=1;
        }
  server {
        include ports.conf;
        server_name abcdefg;
        ssi on;
        proxy_ignore_client_abort on;
        location / {
                gzip on;
                gzip_types text/css application/x-javascript;
                proxy_buffering on;
                proxy_buffer_size 8k;
                proxy_pass   http://abc_pool;
                proxy_set_header Host $host;
                proxy_set_header  X-Real-IP  $remote_addr;
                proxy_set_header  X-Forwarded-For
$proxy_add_x_forwarded_for;
                proxy_cache             abc;
                proxy_cache_valid 200   30m;
                proxy_cache_valid any   0m;
                proxy_cache_use_stale   error timeout invalid_header
updating;
                proxy_cache_key  $scheme$host$uri$is_args$args;
                proxy_hide_header Set-Cookie;
                proxy_hide_header Cookie;
                if ($args ~ module=) {
                        rewrite ^(.*)$ /modulerw/$1 last;
                }
        }
        location ~ (^/ajax/|^/community/|^/forum/|^/newsletter/) {
                proxy_ignore_client_abort on;
                gzip on;
                gzip_types text/css application/x-javascript;
                proxy_buffering off;
                proxy_buffer_size 8k;
                proxy_pass   http://ip_hash_pool;
                proxy_set_header Host $host;
                proxy_pass_header Set-Cookie;
                proxy_pass_header Cookie;
        }
        location /css/ {
               gzip on;
               gzip_types text/css;
                expires 3600;
               root /home/abc/trunk/public;
        }
}

----

Posted at Nginx Forum: http://forum.nginx.org/read.php?2,184782,184782#msg-184782