nginx crashing w/ core dump under moderate load

Philip Jacob phil at stylefeeder.com
Thu Nov 18 23:52:17 MSK 2010


Forgot the config file.

phil.

user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;

worker_rlimit_nofile 65535;
working_directory /opt/local/nginx/cores/;
worker_rlimit_core 500M;

events {
   worker_connections  10240;
}

http {

	server_tokens off;

   include       mime.types;
   default_type  application/octet-stream;

   log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                     '$status $body_bytes_sent "$http_referer" '
                     '"$http_user_agent" "$http_x_forwarded_for"';

   access_log  logs/access.log  main;

   sendfile        on;
   #tcp_nopush     on;

   #keepalive_timeout  0;
   keepalive_timeout  5;

   gzip  on;

   # Only for pre-production testing
   include xxx-network.include;

   proxy_cache_path  /var/www/cache levels=1:2 keys_zone=myhostname-cache:8m max_size=1000m inactive=1m;
   proxy_temp_path /var/www/cache/tmp;
   proxy_cache_methods GET;

   server {
       listen       80 default;
       server_name  www.myhostname.com secure.myhostname.com www.myhostname.com.xxx.net;

	    proxy_cache_use_stale updating;
	    proxy_cache_valid 200 1m; 
       proxy_cache_key "$scheme://$proxy_host$request_uri $cookie_a $cookie_JSESSIONID $cookie_b $cookie_c";
       proxy_buffer_size 64k;
       proxy_buffers 32 64k;

       charset utf-8;

       access_log  logs/www.myhostname.com.access.log  main;

       # This turns out to be critically important.  Otherwise, it sends Nginx in a loop on the homepage
       # and it SSIs the homepage into itself over 200 times.  This seems like a bug in Nginx, so be aware 
       # of it during upgrades.
       location /foo {
       	ssi off;
       	proxy_pass http://127.0.0.1:8080/foo;
       }

       location / {
	    	ssi on;
	    	ssi_silent_errors off;
	    	log_subrequest on;
	    	
	    	########################################################################################
			# NOTE / IMPORTANT / URGENT / README / ATTENTION / ACHTUNG / ATTENZIONE / 
	    	########################################################################################
			# 
			# These settings MUST be changed together.  If you turn caching off by commenting out the 
			# proxy cache directive, you MUST to change X-Caching-Mode to "Off".  Valid values for this 
			# header are On/Off.  FooCode, foo.jsp, the /foo controller and the 
			# esi.tag all rely on this being handled properly.
			# 

	    	proxy_set_header X-Caching-Mode On;
	    	proxy_cache myhostname-cache;

			#
	    	########################################################################################

	    	
	    	proxy_pass http://127.0.0.1:8080/;
           root   html;
           index  index.html index.htm;

		    # For the ESI-like webhit-recording servlet to know what the original URI is.
		    proxy_set_header X-Original-Request-URI $request_uri;
		    proxy_set_header X-Original-Request-Method $request_method;

		    # First added for spring security to redirect correctly for protected pages
		    proxy_set_header Host $http_host;
			
		    # http://wiki.nginx.org/NginxLikeApache
		    proxy_set_header X-Forwarded-Host $host;
		    proxy_set_header X-Forwarded-Server $host;
		    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 
		   
			# Make things served up by this nginx instance totally non-cacheable to the outside world.
			# This is complex and has implications if you change it.  Discuss laboriously with your teammates.
			expires epoch;
	
		    # Pre-launch stuff.
	        # error_page 403 http://x.myhostname.com/;
	
		    satisfy any;
	
		    # For blocking access to everything except the xxx net.
		    # include xxx-network.include; <-- this file format needs redone.
	
		    # Open up to xxx
			include xxx-network.include;
	
		    # Open up to xxx
			allow xxx;
			allow xxx;
	
		    # Leave these in always - the local net and xxx.
			allow xxx;
			allow xxx;
			
			# xxx box for load testing
			allow xxx;
	
		    # Otherwise block.
			deny all;
	
		    # Put a password on stuff, per xxx.
			auth_basic "myhostname";
			auth_basic_user_file .htpasswd;
	 
	        # redirect server error pages to the static page /50x.html
	        #
			error_page   500 502 503 504  /50x.html;
				location = /50x.html {
				root   html;
			}

		}
   }


   # myhostname.com - perm redirect to www
   server { 
		server_name myhostname.com; 
		rewrite ^/(.*) http://www.myhostname.com/$1 permanent; 
   }

   # static assets
   server {
		server_name semi-1.prod.xxx.net xxx.prod.xxx.net;
		root /var/www/domains/prod.xxx.net/semi-1/htdocs/;
		expires max;
				
		location ~ /.svn/* {
		    deny  all;
		}
		
   }

}

On Nov 18, 2010, at 2:55 PM, Philip Jacob wrote:

> I'm experiencing a problem with Nginx 0.8.53 crashing under moderate load.  It's configured as a proxy with ssi enabled.  Config file, backtrace, et al are below (the only modifications are hostnames to protect the innocent).  I don't have a debug log, but I have a lot of other useful info.
> 
> phil.
> 
> ##################################################################
> 2010/11/16 14:55:19 [alert] 7568#0: worker process 7569 exited on signal 11 (core dumped)
> 
> #################################################################### 
> [phil at web01 nginx]$ /opt/local/nginx/sbin/nginx -V
> nginx version: nginx/0.8.53
> built by gcc 4.1.2 20080704 (Red Hat 4.1.2-48)
> configure arguments: --prefix=/opt/local/nginx --with-http_realip_module
> 
> #################################################################### 
> sudo gdb /opt/local/nginx/sbin/nginx /tmp/core.27113 
> [sudo] password for phil: 
> GNU gdb (GDB) Red Hat Enterprise Linux (7.0.1-23.el5_5.2)
> Copyright (C) 2009 Free Software Foundation, Inc.
> License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
> This is free software: you are free to change and redistribute it.
> There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
> and "show warranty" for details.
> This GDB was configured as "i386-redhat-linux-gnu".
> For bug reporting instructions, please see:
> <http://www.gnu.org/software/gdb/bugs/>...
> Reading symbols from /opt/local/nginx-0.8.53/sbin/nginx...done.
> 
> warning: .dynamic section for "/lib/libc.so.6" is not at the expected address
> 
> warning: difference appears to be caused by prelink, adjusting expectations
> Reading symbols from /lib/libcrypt.so.1...(no debugging symbols found)...done.
> Loaded symbols for /lib/libcrypt.so.1
> Reading symbols from /lib/libpcre.so.0...(no debugging symbols found)...done.
> Loaded symbols for /lib/libpcre.so.0
> Reading symbols from /lib/libcrypto.so.6...(no debugging symbols found)...done.
> Loaded symbols for /lib/libcrypto.so.6
> Reading symbols from /usr/lib/libz.so.1...(no debugging symbols found)...done.
> Loaded symbols for /usr/lib/libz.so.1
> Reading symbols from /lib/libc.so.6...(no debugging symbols found)...done.
> Loaded symbols for /lib/libc.so.6
> Reading symbols from /lib/libdl.so.2...(no debugging symbols found)...done.
> Loaded symbols for /lib/libdl.so.2
> Reading symbols from /lib/ld-linux.so.2...(no debugging symbols found)...done.
> Loaded symbols for /lib/ld-linux.so.2
> Reading symbols from /lib/libnss_files.so.2...(no debugging symbols found)...done.
> Loaded symbols for /lib/libnss_files.so.2
> Core was generated by `nginx: worker process                                          '.
> Program terminated with signal 11, Segmentation fault.
> #0  ngx_rbtree_min (tree=0xb778c040, node=0xb778d380) at src/core/ngx_rbtree.h:75
> 75	    while (node->left != sentinel) {
> (gdb) bt
> #0  ngx_rbtree_min (tree=0xb778c040, node=0xb778d380) at src/core/ngx_rbtree.h:75
> #1  ngx_rbtree_delete (tree=0xb778c040, node=0xb778d380) at src/core/ngx_rbtree.c:178
> #2  0x080856e0 in ngx_http_file_cache_free (c=0x85bbe9c, tf=0x0) at src/http/ngx_http_file_cache.c:941
> #3  0x0807be76 in ngx_http_upstream_finalize_request (r=0x85c7fa0, u=0x85bbc0c, rc=499) at src/http/ngx_http_upstream.c:3002
> #4  0x0807e005 in ngx_http_upstream_next (r=0x85c7fa0, u=0x85bbc0c, ft_type=4) at src/http/ngx_http_upstream.c:2834
> #5  0x0807e0fb in ngx_http_upstream_process_header (r=0x85c7fa0, u=0x85bbc0c) at src/http/ngx_http_upstream.c:1459
> #6  0x0807c965 in ngx_http_upstream_handler (ev=0x0) at src/http/ngx_http_upstream.c:895
> #7  0x0805df3f in ngx_event_expire_timers () at src/event/ngx_event_timer.c:149
> #8  0x0805de76 in ngx_process_events_and_timers (cycle=0x85b5e08) at src/event/ngx_event.c:261
> #9  0x08063913 in ngx_worker_process_cycle (cycle=0x85b5e08, data=0x0) at src/os/unix/ngx_process_cycle.c:795
> #10 0x08062201 in ngx_spawn_process (cycle=0x85b5e08, proc=0x806385b <ngx_worker_process_cycle>, data=0x0, 
>    name=0x80a3a81 "worker process", respawn=-3) at src/os/unix/ngx_process.c:196
> #11 0x08062f22 in ngx_start_worker_processes (cycle=0x85b5e08, n=1, type=-3) at src/os/unix/ngx_process_cycle.c:355
> #12 0x08063e97 in ngx_master_process_cycle (cycle=0x85b5e08) at src/os/unix/ngx_process_cycle.c:136
> #13 0x0804b4ff in main (argc=3, argv=0xbfd5b294) at src/core/nginx.c:401
> 
> 
> 





More information about the nginx mailing list