out of tcptw
Evgenii Davidov
dado на korolev-net.ru
Ср Дек 7 07:44:35 UTC 2011
товарищи ученые
вчера я на 5 минут выбежал из tcptw:
ITEM SIZE LIMIT USED FREE REQUESTS FAILURES
tcptw: 52, 40968, 11922, 22638, 47803752, 0
tcptw: 52, 40968, 31451, 3109, 47835019, 0
tcptw: 52, 40968, 39130, 686, 47873887, 0
tcptw: 52, 40968, 40335, 633, 47889337, 26837
tcptw: 52, 40968, 40397, 571, 47889399, 69815
tcptw: 52, 40968, 40465, 503, 47889471, 112430
tcptw: 52, 40968, 40522, 446, 47889530, 153805
tcptw: 52, 40968, 35654, 5314, 47904332, 173111
tcptw: 52, 40968, 27982, 12986, 47932049, 173111
stub status при этом показывал:
Reading: 351 Writing: 50 Waiting: 6801
Reading: 649 Writing: 78 Waiting: 6502
Reading: 880 Writing: 152 Waiting: 6457
Reading: 856 Writing: 209 Waiting: 6238
Reading: 854 Writing: 167 Waiting: 6120
Reading: 579 Writing: 43 Waiting: 6421
Reading: 443 Writing: 41 Waiting: 6368
как вы думаете, правильно ли я понимаю что это произошло потому что было настроено:
worker_processes 4;
worker_connections 2000;
и при приближении к 8000 nginx стал в срочном порядке закрывать соединения и этих закрытых соединений в состоянии протухания стало слишком много?
(по моим прикидкам клиентов было тыщ 11)
также было настроено
sysctl net.inet.tcp.fast_finwait2_recycle=1
sysctl net.inet.tcp.finwait2_timeout=5000
за эти 5 минут успел 500 раз ругнуться named типа: client 84.53.200.24#1202: error sending response: not enough free resources
7.2-STABLE i386
nginx/1.0.2
apache:
2 requests currently being processed, 1 idle servers
top -I :
last pid: 16701; load averages: 1.41, 0.84, 0.58 up 24+07:33:33 01:39:00
70 processes: 2 running, 68 sleeping
Mem: 606M Active, 2327M Inact, 275M Wired, 146M Cache, 112M Buf, 154M Free
Swap: 8192M Total, 168K Used, 8192M Free
PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU COMMAND
676 football 1 4 0 110M 4784K kqread 3 376:25 28.37% nginx
677 football 1 4 0 110M 4568K kqread 3 356:35 25.59% nginx
674 football 1 4 0 110M 4276K kqread 3 294:21 13.09% nginx
675 football 1 4 0 109M 3900K kqread 0 272:31 5.76% nginx
13234 www 1 4 0 16904K 10276K accept 3 0:26 4.05% httpd
15537 www 1 4 0 17012K 10180K accept 0 0:18 3.86% httpd
15260 www 1 4 0 17776K 11124K accept 3 0:15 3.37% httpd
621 mysql 16 20 0 328M 202M kserel 3 138:41 1.56% mysqld
vmstat -z :
UMA Kegs: 128, 0, 82, 8, 82, 0
UMA Zones: 480, 0, 82, 6, 82, 0
UMA Slabs: 64, 0, 2719, 585, 123228, 0
UMA RCntSlabs: 104, 0, 6087, 18, 137964, 0
UMA Hash: 128, 0, 3, 27, 7, 0
16 Bucket: 76, 0, 26, 74, 111, 0
32 Bucket: 140, 0, 66, 102, 246, 0
64 Bucket: 268, 0, 43, 97, 497, 82
128 Bucket: 524, 0, 2330, 267, 167059, 16334
VM OBJECT: 128, 0, 90686, 934, 60252229, 0
MAP: 140, 0, 7, 21, 7, 0
KMAP ENTRY: 68, 57456, 25, 2775, 2830203, 0
MAP ENTRY: 68, 0, 1908, 3020, 266844593, 0
DP fakepg: 72, 0, 0, 212, 20, 0
mt_zone: 1032, 0, 181, 2, 181, 0
16: 16, 0, 2692, 1165, 233417157, 0
32: 32, 0, 1942, 1787, 240579919, 0
64: 64, 0, 4304, 6493, 388300570, 0
128: 128, 0, 2048, 2002, 78236147, 0
256: 256, 0, 878, 1147, 61735665, 0
512: 512, 0, 504, 1896, 41332494, 0
1024: 1024, 0, 52, 748, 43001389, 0
2048: 2048, 0, 174, 912, 10085, 0
4096: 4096, 0, 134, 701, 31993215, 0
Files: 76, 0, 8440, 660, 911860153, 0
TURNSTILE: 76, 0, 428, 148, 428, 0
umtx pi: 52, 0, 0, 0, 0, 0
PROC: 704, 0, 93, 157, 1514550, 0
THREAD: 572, 0, 268, 159, 928968, 0
UPCALL: 44, 0, 15, 375, 43, 0
SLEEPQUEUE: 32, 0, 428, 363, 428, 0
VMSPACE: 236, 0, 56, 200, 1514513, 0
cpuset: 40, 0, 2, 182, 2, 0
audit_record: 864, 0, 0, 0, 0, 0
mbuf_packet: 256, 0, 1359, 817, 2417792721, 0
mbuf: 256, 0, 8758, 976, 4193507441, 0
mbuf_cluster: 2048, 200000, 2176, 770, 127104, 0
mbuf_jumbo_pagesize: 4096, 12800, 4031, 583, 281822622, 0
mbuf_jumbo_9k: 9216, 6400, 0, 0, 0, 0
mbuf_jumbo_16k: 16384, 3200, 0, 0, 0, 0
mbuf_ext_refcnt: 4, 0, 4050, 822, 568097356, 0
ACL UMA zone: 388, 0, 0, 0, 0, 0
g_bio: 132, 0, 0, 3306, 158378233, 0
ata_request: 192, 0, 0, 1700, 35106790, 0
ata_composite: 184, 0, 0, 0, 0, 0
VNODE: 276, 0, 89870, 10440, 203908212, 0
VNODEPOLL: 64, 0, 4, 173, 5, 0
S VFS Cache: 68, 0, 94941, 10003, 207282902, 0
L VFS Cache: 291, 0, 588, 2103, 4320525, 0
NAMEI: 1024, 0, 0, 656, 1813579725, 0
DIRHASH: 1024, 0, 1834, 726, 1371844, 0
pipe: 396, 0, 56, 724, 711044, 0
ksiginfo: 80, 0, 221, 115, 221, 0
itimer: 220, 0, 0, 0, 0, 0
KNOTE: 68, 0, 10633, 791, 1248327164, 0
socket: 416, 204804, 10679, 634, 132615432, 0
unpcb: 168, 204815, 79, 588, 2036358, 0
ipq: 32, 6328, 0, 565, 380793, 0
udp_inpcb: 180, 204820, 3, 217, 527580, 0
udpcb: 8, 204827, 3, 809, 527580, 0
inpcb: 180, 204820, 50932, 680, 130051395, 0
tcpcb: 464, 204800, 10597, 635, 130051395, 0
tcptw: 52, 40968, 40335, 633, 47889337, 26837
syncache: 104, 15392, 325, 748, 119415191, 0
hostcache: 76, 15400, 8969, 2081, 14104552, 0
tcpreass: 20, 12506, 1, 844, 414209, 0
sackhole: 20, 0, 55, 959, 29266551, 0
sctp_ep: 816, 25600, 0, 0, 0, 0
sctp_asoc: 1436, 40000, 0, 0, 0, 0
sctp_laddr: 24, 80040, 0, 290, 2, 0
sctp_raddr: 388, 80000, 0, 0, 0, 0
sctp_chunk: 96, 400000, 0, 0, 0, 0
sctp_readq: 76, 400000, 0, 0, 0, 0
sctp_stream_msg_out: 64, 400020, 0, 0, 0, 0
sctp_asconf: 24, 400055, 0, 0, 0, 0
sctp_asconf_ack: 24, 400055, 0, 0, 0, 0
ripcb: 180, 204820, 0, 110, 98, 0
rtentry: 124, 0, 16, 356, 5231, 0
IPFW dynamic rule: 108, 0, 0, 0, 0, 0
SWAPMETA: 276, 121576, 24, 60, 241, 0
Mountpoints: 720, 0, 5, 10, 5, 0
FFS inode: 124, 0, 89835, 10326, 203905281, 0
FFS1 dinode: 128, 0, 0, 0, 0, 0
FFS2 dinode: 256, 0, 89835, 10275, 203905281, 0
nginx.conf:
user foo www;
worker_processes 4;
pid /var/run/nginx.pid;
events {
worker_connections 2000;
use kqueue;
}
http {
proxy_temp_path /var/nginx/tmp ;
proxy_cache_path /var/nginx/cache levels=1:2 keys_zone=cache:90m max_size=90m ;
proxy_cache_valid 60m;
proxy_cache_min_uses 2;
proxy_cache_bypass $cookie_nocache $arg_nocache $http_pragma $http_authorization $arg_f ;
proxy_no_cache $cookie_nocache $arg_nocache $http_pragma $http_authorization $arg_f ;
proxy_store_access user:rw group:rw all:rw;
proxy_cache_use_stale error timeout invalid_header http_500 http_502 http_503 http_504;
client_max_body_size 10m ;
log_format combined1 '$remote_addr $server_name $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"';
error_log /var/log/nginx-error.log warn ;
access_log /dev/null ;
server_names_hash_bucket_size 128 ;
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] '
'"$request" $status $bytes_sent '
'"$http_referer" "$http_user_agent" '
'"$gzip_ratio"';
log_format download '$remote_addr - $remote_user [$time_local] '
'"$request" $status $bytes_sent '
'"$http_referer" "$http_user_agent" '
'"$http_range" "$sent_http_content_range"';
client_header_timeout 10;
client_body_timeout 10;
send_timeout 10;
client_header_buffer_size 1k;
large_client_header_buffers 4 16k;
gzip on;
gzip_min_length 1100;
gzip_buffers 4 8k;
gzip_types text/plain;
output_buffers 1 32k;
postpone_output 1460;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
send_lowat 12000;
keepalive_timeout 50 ;
reset_timedout_connection on;
limit_zone one $binary_remote_addr 16m;
server {
listen x.x.x.x:80 ;
server_name bar ;
location / {
proxy_pass http://127.0.0.1:80/ ;
limit_conn one 10;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
}
location ~* .(jpg|jpeg|gif|png|css|js|ico)$ {
root /home/wit/ ;
expires 30d;
}
location /stub_status {
stub_status on;
allow x.x.x.x/32;
deny all;
}
}
server {
listen x.x.x.x:80 ;
server_name foo ;
location / {
proxy_pass http://127.0.0.1:80/ ;
limit_conn one 10;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Range "";
proxy_set_header Request-Range "";
proxy_buffer_size 4k;
proxy_buffers 4 128k;
proxy_busy_buffers_size 256k;
proxy_temp_file_write_size 256k;
proxy_cache cache;
expires -1 ;
charset windows-1251 ;
}
location ^~ /buttons {
root /home/foo/public_html ;
expires 1h;
}
location ~* .(jpg|jpeg|gif|png|css|js|ico|xml|rss)$ {
root /home/foo/public_html ;
expires 30d;
}
location /stub_status {
stub_status on;
allow x.x.x.x/32;
deny all;
}
}
}
спасибо!
--
Evgenii V Davidov
Подробная информация о списке рассылки nginx-ru