nginx struggling to accept connections during peak load
tempspace
nginx-forum at nginx.us
Sun Sep 29 20:40:05 UTC 2013
Hello,
I had posted to the mailing list earlier this week, but I managed to gather
some new information that points directly to nginx (almost certainly my
configuration), so I thought I'd post something more concise.
I am running edge boxes which use nginx to terminate SSL which passes to
haproxy on the same server. During our peak load time, we are experiencing
intermittent slow connection issues which drives up our response time graphs
from external sources. Every log within our infrastructure shows no
problems, including the edge nginx that we're having issues with.
Today, I was able to setup some boxes from different providers and run some
curl tests in a loop. I setup a bash script that made a curl request to our
edge nginx server for a specific API call. In another bash script, I made a
curl request for the same API call, but bypassing nginx and going directly
to haproxy that is located on the same exact box. By doing this, the curls
to the nginx server showed intermittent big delays in the connection phase
before nginx picks up the phone. The haproxy logs showed absolutely no
issues at all in connecting. Because haproxy is on the same server, I
believe that rules out anything related to a networking issue, both physical
and kernel related.
My SSL connections usually look like this from a cURL:
time_namelookup: 0.001
time_connect: 0.035
time_appconnect: 0.109
time_pretransfer: 0.109
time_redirect: 0.000
time_starttransfer: 0.150
----------
time_total: 0.150
During my peak load, they intermittently (every 3-5 seconds) look like this
(though most of the time, 3 seconds)
time_namelookup: 0.001
time_connect: 9.033
time_appconnect: 9.109
time_pretransfer: 9.109
time_redirect: 0.000
time_starttransfer: 9.148
----------
time_total: 9.148
So, here is my nginx config. I'm running nginx 1.4.1. The system itself
doesn't go beyond 30% CPU combined and all other metrics look good as well.
What can I do better (I'm sure lots)?
user www-data;
worker_processes 11; # 12 cores, 24 with HT
worker_rlimit_nofile 500000;
error_log /var/log/nginx/error.log;
pid /var/run/nginx.pid;
events {
use epoll;
multi_accept off;
accept_mutex off;
worker_connections 65536;
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
proxy_buffering off;
log_format access '$http_x_forwarded_for - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent" "$host" "$request_time"
"$upstream_response_time"';
upstream apiv2-ssl {
server 127.0.0.1:xxxxxx max_fails=3 fail_timeout=15s;
}
upstream api {
server 127.0.0.1:xxxxxx max_fails=3 fail_timeout=15s;
}
upstream secure {
server 127.0.0.1:xxxxxx max_fails=3 fail_timeout=15s;
}
upstream facebook {
server 127.0.0.1:xxxxx max_fails=3 fail_timeout=15s;
}
upstream testing {
server 127.0.0.1:xxxxx max_fails=3 fail_timeout=15s;
}
server {
listen x.x.x.x:443;
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_apiv2.log access;
error_log /var/log/nginx/error_apiv2.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxxx.key;
ssl_session_cache shared:SSLv2:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://apiv2-ssl;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}
server {
listen x.x.x.x:443;
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_apiv3.log access;
error_log /var/log/nginx/error_apiv3.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxx.key;
ssl_session_cache shared:SSLv3:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://api;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
set $msecstart "${msec}000";
if ($msecstart ~ "^(.*)\.(.*)") {set $msecout "t=$1$2";}
proxy_set_header X-Request-Start $msecout;
}
}
server {
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_apiv3.log access;
error_log /var/log/nginx/error_apiv3.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxx.key;
ssl_session_cache shared:SSLv3:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://testing;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}
server {
listen x.x.x.x:443;
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_secure.log access;
error_log /var/log/nginx/error_secure.log;
gzip on;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxxx.key;
ssl_session_cache shared:SSLsecure:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://secure;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}
server {
listen x.x.x.x:443;
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_facebook.log access;
error_log /var/log/nginx/error_facebook.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/xxx.key;
ssl_session_cache shared:SSLfacebook:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://facebook;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}
server {
listen x.x.x.x:443;
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_api.log access;
error_log /var/log/nginx/error_api.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxx.key;
ssl_session_cache shared:SSLapi:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://api;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}
server {
listen x.x.x.x:443;
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access.log access;
error_log /var/log/nginx/error.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxx.key;
ssl_session_cache shared:SSLv3:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://facebook;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}
}
Posted at Nginx Forum: http://forum.nginx.org/read.php?2,243245,243245#msg-243245
More information about the nginx
mailing list