Nginx capping at 320 requests per second

[dpn]

Hey, we run a website of fairly decent volume.. up to nearly 4m
pageviews a day.

At the moment we run a single machine with nginx and mysql and two
worker machines with memcached and tornado instances. The nginx server
is a reverse proxy to the workers and also serves static media.

The CPU load and memory usage on both of the worker boxes are well
within reasonable expectations.

What I am observing is that nginx gets to about 320 requests per second
then requests start backing up. Sometimes taking the server down, see
this image: http://dl.dropbox.com/u/367355/nginx.png

When the server doesn't go down, we see a flattening of requests around
the 320 mark, and the number of "waiting" requests and the memory usage
of nginx spikes considerably.

I've tried upping the number of workers in case all of them are blocking
for long enough to cause this cascading effect (the tornado db driver is
not async) but didn't really see an improvement by adding more. I've
also added lots of async memcached access to avoid hitting the db too
much.

I've included the configs below.. thanks for any help you may have!

[code]
user www-data;
worker_processes 4;
worker_rlimit_nofile 32768;


error_log  /dev/null crit;
pid        /var/run/nginx.pid;

events {
    worker_connections  8192;
    use epoll;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    access_log	/dev/null;

    sendfile        on;

    keepalive_timeout  0;
    tcp_nodelay        on;

    gzip  on;
    gzip_types text/css text/plain text/javascript
application/x-javascript application/json;
    gzip_comp_level 5;
    gzip_disable     "msie6";
 
    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

[/code]


[code]
upstream bar {
 server worker1:8888 max_fails=1 fail_timeout=10s;
 server worker2:8888 max_fails=1 fail_timeout=10s;
}


server { # simple reverse-proxy
    listen       80;
    server_name  bar.net;
    #access_log   logs/bar.access.log;
    access_log /dev/null;


    location /nginx_status {
            stub_status on;
            access_log off;
            allow 127.0.0.1;
            deny all;
    }

    location ^~ /static/ {
        root /home/foo/bar;
        if ($query_string) {
                expires max;
            }
        }

    # pass requests for dynamic content to tornado
    location / {
            proxy_pass_header Server;
            proxy_set_header Host $http_host;
            proxy_redirect false;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Scheme $scheme;
            proxy_pass      http://tweete;
    }
    error_page 411 /411.html;
    location = /411.html {
         root  /home/foo/bar/static/error;
    }

    error_page 500 502 503 504  /500.html;
    location = /500.html {
         root  /home/foo/bar/static/error;
     }
  }

[/code]

Posted at Nginx Forum: http://forum.nginx.org/read.php?2,123754,123754#msg-123754