nginx reload not working, restart creates orphan yet active worker

[groknaut]

hello,

after trying to reload nginx after making a config change, the reload
didn't show that the new config had taken affect. i then issued a
restart and what happened was, the nginx parent died, yet the nginx
worker kept running serving requests. any help?

this appears to happen after the nginx parent has been running for "a
while".

we're running nginx 1.0.5 with the following patches from
https://github.com/yaoweibin:
* nginx_ajp_module
* nginx_upstream_module
* nginx_syslog_patch

i have a pile of debug error log, lsof, process list, and straces. shall
i provide those? if so, where? off-list?

the config change i was making was node membership in the pool in
upstream.conf - taking one node in and out.


processes prior to restart:

root     30403  0.0  0.2  13972  4912 ?        Ss   Aug17   0:01 nginx:
master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
nginx     9685  0.0  0.2  14364  4220 ?        S    Oct13   3:53  \_
nginx: worker process


nginx.conf:


user              nginx;
worker_processes  1;
syslog local2 nginx;
error_log  /var/log/nginx/error.log debug;
pid        /var/run/nginx.pid;
events {
    worker_connections  1024;
}
http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    proxy_buffers 8 16k;
    proxy_buffer_size 32k;
    log_format  main  '$remote_addr - $remote_user [$time_local]
"$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  syslog:warn|/var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;
    gzip  on;

    server {
        listen       80;
        server_name  _;
        location /nginx-status {
            stub_status on;
            access_log off;
            allow 10.0.0.0/8;
            allow 127.0.0.1;
            deny all;
        }
        location /upstream-status {
            check_status;
            access_log off;
            allow 10.0.0.0/8;
            allow 127.0.0.1;
            deny all;
        }

        error_page  404     /404.html;
        location = /404.html {
            root   /usr/share/nginx/error;
        }
        error_page  403     /403.html;
        location = /403.html {
            root   /usr/share/nginx/error;
        }
        error_page  500 502 504   /500.html;
        location = /500.html {
            root   /usr/share/nginx/error;
        }
        error_page  503     /503.html;
        location = /503.html {
            root   /usr/share/nginx/error;
        }
        if ($request_filename ~* "api/(.*)") {
            return 403;
            break;
        }
        set $global_ssl_redirect 'yes';
        if ($request_filename ~ "nginx-status") {
            set $global_ssl_redirect 'no';
        }
        if ($request_filename ~ "upstream-status") {
            set $global_ssl_redirect 'no';
        }
        if ($global_ssl_redirect ~* '^yes$') {
            rewrite ^ https://$host$request_uri? permanent;
            break;
        }
    }
    include upstream.conf;

    server {
        listen  443;
        server_name  _;
        ssl on;
        ssl_certificate certs/wildcard.sushimysavior.com.crt;
        ssl_certificate_key certs/wildcard.sushimysavior.com.key;
        ssl_protocols SSLv3 TLSv1;
        ssl_ciphers HIGH:!ADH:!MD5;
        set_real_ip_from 10.0.0.0/8;
        real_ip_header X-Forwarded-For;
        add_header Cache-Control public;
        location /WEB-INF {
            deny all;
        }
        location /jmxb {
            deny all;
        }
        location /static {
            root /var/www;
            expires 0;
            add_header Pragma "no-cache";
            add_header Cache-Control "no-cache";
            if ($request_filename ~ "Plugin.exe") {
                add_header Content-Disposition "attachment;
filename=Plugin.exe";
            }
        }
        location / {
            proxy_pass http://tomcats_http;
            proxy_connect_timeout 10s;
            proxy_next_upstream error timeout invalid_header http_500
http_503 http_502 http_504;
            proxy_set_header Host $host;
            proxy_set_header X-Server-Port $server_port;
            proxy_set_header X-Server-Protocol https;
            proxy_set_header X-Forwarded-For
$proxy_add_x_forwarded_for;
            proxy_set_header Strict-Transport-Security
max-age=315360000;
            proxy_set_header X-Secure true;
            proxy_set_header Transfer-Encoding "";
            if ($request_uri ~* "\.(ico|css|js|gif|jpe?g|png)") {
                expires 365d;
                break;
            }
        }

        error_page  404     /404.html;
        location = /404.html {
            root   /usr/share/nginx/error;
        }
        error_page  403     /403.html;
        location = /403.html {
            root   /usr/share/nginx/error;
        }
        error_page  500 502 504   /500.html;
        location = /500.html {
            root   /usr/share/nginx/error;
        }
        error_page  503     /503.html;
        location = /503.html {
            root   /usr/share/nginx/error;
        }
    }
}


upstream.conf:

upstream tomcats_http {
    server app2:1802;
    server app4:1802;
    check interval=3000 rise=3 fall=2 timeout=1000 type=http
default_down=false;
    check_http_send "GET /monitor/ystatus HTTP/1.0\r\n\r\n";
}

Posted at Nginx Forum: http://forum.nginx.org/read.php?2,216933,216933#msg-216933