Worker processes not shutting down

Tue Feb 12 07:01:55 UTC 2013

Hi,

I have the same issue and would like to find a solution. This is in
production so couple of 3rd party module compiled. This happens when we
reload the nginx config.

# ps -ef | grep nginx
root     10163     1  0 Feb08 ?        00:00:00 nginx: master process
/usr/sbin/nginx -c /etc/nginx/nginx.conf
nginx    14434 10163  0 06:39 ?        00:00:00 nginx: worker process is
shutting down
nginx    15664 10163  0 06:40 ?        00:00:00 nginx: worker process
nginx    15665 10163  0 06:40 ?        00:00:00 nginx: worker process
nginx    15666 10163  0 06:40 ?        00:00:00 nginx: worker process
nginx    15667 10163  0 06:40 ?        00:00:00 nginx: worker process
root     17489  9311  0 06:43 pts/3    00:00:00 grep nginx
nginx    23887 10163  0 Feb08 ?        00:00:12 nginx: worker process is
shutting down
nginx    23888 10163  0 Feb08 ?        00:00:08 nginx: worker process is
shutting down
nginx    23892 10163  0 Feb08 ?        00:00:20 nginx: worker process is
shutting down
nginx    32240 10163  0 Feb11 ?        00:00:15 nginx: worker process is
shutting down
nginx    32241 10163  0 Feb11 ?        00:00:16 nginx: worker process is
shutting down
nginx    32244 10163  0 Feb11 ?        00:00:13 nginx: worker process is
shutting down
nginx    32245 10163  0 Feb11 ?        00:00:19 nginx: worker process is
shutting down

# pstack 32245
#0  0x00000039dced4863 in __epoll_wait_nocancel () from /lib64/libc.so.6
#1  0x0000000000425e57 in ngx_epoll_process_events ()
#2  0x000000000041cc2e in ngx_process_events_and_timers ()
#3  0x0000000000423d7d in ngx_worker_process_cycle ()
#4  0x000000000042236d in ngx_spawn_process ()
#5  0x00000000004232cc in ngx_start_worker_processes ()
#6  0x0000000000424a9e in ngx_master_process_cycle ()
#7  0x00000000004069fd in main ()

]# pstack 32244
#0  0x00000039dced4863 in __epoll_wait_nocancel () from /lib64/libc.so.6
#1  0x0000000000425e57 in ngx_epoll_process_events ()
#2  0x000000000041cc2e in ngx_process_events_and_timers ()
#3  0x0000000000423d7d in ngx_worker_process_cycle ()
#4  0x000000000042236d in ngx_spawn_process ()
#5  0x00000000004232cc in ngx_start_worker_processes ()
#6  0x0000000000424a9e in ngx_master_process_cycle ()
#7  0x00000000004069fd in main ()

# pstack 32241
#0  0x00000039dced4863 in __epoll_wait_nocancel () from /lib64/libc.so.6
#1  0x0000000000425e57 in ngx_epoll_process_events ()
#2  0x000000000041cc2e in ngx_process_events_and_timers ()
#3  0x0000000000423d7d in ngx_worker_process_cycle ()
#4  0x000000000042236d in ngx_spawn_process ()
#5  0x00000000004232cc in ngx_start_worker_processes ()
#6  0x0000000000424a9e in ngx_master_process_cycle ()
#7  0x00000000004069fd in main ()

# strace  -p 32244
Process 32244 attached - interrupt to quit
epoll_wait(21, 

# strace  -p 32241
Process 32241 attached - interrupt to quit
epoll_wait(19, 

# strace  -p 32245
Process 32245 attached - interrupt to quit
epoll_wait(24, {{EPOLLIN|EPOLLOUT, {u32=412706096, u64=412706096}}}, 512,
9351728) = 1
recvfrom(447, "HTTP/1.1 200 OK\r\nDate: Tue, 12 F"..., 2048, 0, NULL, NULL)
= 611
write(444,
"\27\3\1\2\200\224T\204\354\353w\344+\267c9\314\270\30I\216\200\314\354\376\3\323\202\332(:\323"...,
645) = 645
recvfrom(447, 0x1a0e98f0, 2048, 0, 0, 0) = -1 EAGAIN (Resource temporarily
unavailable)
epoll_wait(24, {{EPOLLIN|EPOLLOUT, {u32=412730864, u64=412730864}}}, 512,
9351656) = 1
read(443, "\27\3\1\0
\t\355\205\202\267>\274\240\324qR\305\334\36\223\355\0251\31[A\366\225~\217\220\252"...,
34821) = 698
read(443, 0x1a0d0600, 34821)            = -1 EAGAIN (Resource temporarily
unavailable)
sendto(446, "GET /CometServer/cometd/connect?"..., 621, 0, NULL, 0) = 621
epoll_wait(24, {{EPOLLIN|EPOLLOUT, {u32=412783472, u64=412783472}}}, 512,
9351648) = 1
recvfrom(446, "HTTP/1.1 200 OK\r\nDate: Tue, 12 F"..., 1024, 0, NULL, NULL)
= 1024
write(443, "\27\3\1\4
+T@\376\260\306q\vrq\1\240T\244\377\227\23g\5\340\262FYs:]o"..., 1061) =
1061
recvfrom(446, "|TRT\\\"},\\\"DAT\\\":{\\\"DQ\\\":[\\\"PFX|1"..., 1024, 0,
NULL, NULL) = 1024
write(443, "\27\3\1\4
\262\202\372V\326\251S\252?\353\266\307\272\257\240\306\247\205\345\307\320L\31W\223\337h"...,
1061) = 1061
recvfrom(446, "\":\"{\\\"RT\\\":\\\"4\\\",\\\"HED\\\":{\\\"DQ\\\""..., 1024,
0, NULL, NULL) = 1024
write(443, "\27\3\1\4
9(\241\235\377\350\220\4\320&\25\34|;#\215\212\220\"\263\vFz\350\360ks"...,
1061) = 1061
recvfrom(446, "a\":{\"scope\":\"public\",\"server\":\"P"..., 1024, 0, NULL,
NULL) = 1024

# cat /etc/redhat-release
Red Hat Enterprise Linux Server release 5.7 (Tikanga)

# uname -rop
2.6.18-274.el5 x86_64 GNU/Linux

# nginx -V
nginx version: nginx/1.2.6
built by gcc 4.1.2 20080704 (Red Hat 4.1.2-50)
TLS SNI support disabled
configure arguments: --prefix=/etc/nginx/ --sbin-path=/usr/sbin/nginx
--conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log
--http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid
--lock-path=/var/run/nginx.lock
--http-client-body-temp-path=/var/cache/nginx/client_temp
--http-proxy-temp-path=/var/cache/nginx/proxy_temp
--http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp
--http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp
--http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx
--with-http_ssl_module --with-http_realip_module --with-http_addition_module
--with-http_sub_module --with-http_dav_module --with-http_flv_module
--with-http_gzip_static_module --with-http_random_index_module
--with-http_secure_link_module --with-http_stub_status_module
--with-file-aio --without-mail_pop3_module --without-mail_imap_module
--without-mail_smtp_module --with-debug
--add-module=/usr/local/hirantha/rpmbuild/BUILD/nginx-1.2.6/contrib/ngx_devel_kit-master
--add-module=/usr/local/hirantha/rpmbuild/BUILD/nginx-1.2.6/contrib/echo-nginx-module-master
--add-module=/usr/local/hirantha/rpmbuild/BUILD/nginx-1.2.6/contrib/set-misc-nginx-module-master
--add-module=/usr/local/hirantha/rpmbuild/BUILD/nginx-1.2.6/contrib/srcache-nginx-module-master
--add-module=/usr/local/hirantha/rpmbuild/BUILD/nginx-1.2.6/contrib/nginx-sticky-module-1.1
--add-module=/usr/local/hirantha/rpmbuild/BUILD/nginx-1.2.6/contrib/nginx_upstream_check_module-master
--add-module=/usr/local/hirantha/rpmbuild/BUILD/nginx-1.2.6/contrib/memc-nginx-module-master
--add-module=/usr/local/hirantha/rpmbuild/BUILD/nginx-1.2.6/contrib/nginx_cross_origin_module-master
--add-module=/usr/local/hirantha/rpmbuild/BUILD/nginx-1.2.6/contrib/nginx_tcp_proxy_module-master
--add-module=/usr/local/hirantha/rpmbuild/BUILD/nginx-1.2.6/contrib/naxsi-core-0.48/naxsi_src
--with-cc-opt='-O2 -g -m64 -mtune=generic'

Thanks in advance.

Posted at Nginx Forum: http://forum.nginx.org/read.php?2,221591,236123#msg-236123