nginx crash

MagicBear magicbearmo at gmail.com
Sat Nov 12 14:07:22 UTC 2011


happen on a of main upstream server dead.

Here is the config

proxy_cache_path /dev/shm/cdn_cache_comment  levels=1:2
keys_zone=cache_comment_mem:32m max_size=128m;
limit_req_zone  $binary_remote_addr  zone=limit_comment:16m   rate=50r/s;

upstream backend_comment {
	server 10.0.0.1 weight=10 fail_timeout=30s;
	server 10.0.0.2 backup weight=5 fail_timeout=30s;
	keepalive 30;
}
map $http_accept_encoding $gzip {
	default "";
	~gzip " gzip";
}
server {
	listen 80;
	server_name example.com;
	
	access_log /dev/shm/logger logger;	
	error_log /dev/null;
	
	limit_req zone=limit_comment burst=200;
	
	gzip_buffers     4 16k;
	gzip_min_length  1k;
	gzip_http_version 1.1;
	gzip_comp_level 9;
	gzip_types text/plain application/x-javascript text/css application/xml;
	gzip_vary on;
	gzip_proxied any;
	
	proxy_ssl_session_reuse on;
	proxy_cache_min_uses 1;
	proxy_connect_timeout 5s;
	proxy_send_timeout   60s;
	proxy_read_timeout   60s;
	
	gzip on;
	
	location ~ ^/purge(/.*) {
		proxy_cache_purge cache_comment_mem $1$is_args$args;
	}
	location = /crossdomain.xml
	{
		proxy_cache_min_uses 1;
		proxy_cache cache_comment_mem;
		proxy_cache_valid 200 302 1M;
		proxy_cache_key $uri$is_args$args$gzip;
		proxy_hide_header X-Cache;
		proxy_hide_header X-Via;
		add_header X-Cache "$upstream_cache_status from $hostname";
		add_header X-Cache-Lookup $upstream_http_x_cache;
		gzip on;
		expires max;
		real_ip_header X-Real-IP;
		set_real_ip_from	0.0.0.0/0;
		proxy_set_header X-Cache-Server $hostname;
		proxy_set_header X-Real-IP $remote_addr;
		proxy_set_header Connection "keep-alive";
		proxy_set_header Host example.com;
		proxy_hide_header X-Cache;
		proxy_hide_header X-Via;
		proxy_pass http://backend_comment;
		proxy_next_upstream error timeout invalid_header http_500 http_502
http_503 http_504;
	}
	location ~ ^/dm,
	{
		proxy_cache_min_uses 1;
		proxy_cache cache_comment_mem;
		proxy_cache_valid 200 302 5s;
		proxy_cache_key $uri$gzip;
		proxy_hide_header X-Cache;
		proxy_hide_header X-Via;
		add_header X-Cache "$upstream_cache_status from $hostname";
		add_header X-Cache-Lookup $upstream_http_x_cache;
		gzip on;
		expires 15s;
		real_ip_header X-Real-IP;
		set_real_ip_from	0.0.0.0/0;
		proxy_set_header X-Cache-Server $hostname;
		proxy_set_header X-Real-IP $remote_addr;
		proxy_set_header Connection "keep-alive";
		proxy_set_header Host example.com;
		proxy_hide_header X-Cache;
		proxy_hide_header X-Via;
		proxy_pass http://backend_comment;
		proxy_next_upstream error timeout invalid_header http_500 http_502
http_503 http_504;
	}
	location /
	{
		proxy_buffering off;	#Direct send to client
		add_header X-Cache "BYPASS from $hostname";
		add_header X-Cache-Lookup $upstream_http_x_cache;
		gzip on;
		expires -1;
		real_ip_header X-Real-IP;
		set_real_ip_from	0.0.0.0/0;
		proxy_set_header X-Cache-Server $hostname;
		proxy_set_header X-Real-IP $remote_addr;
		proxy_set_header Connection "keep-alive";
		proxy_set_header Host example.com;
		proxy_hide_header X-Cache;
		proxy_hide_header X-Via;
		proxy_pass http://backend_comment;
		proxy_next_upstream error timeout invalid_header http_500 http_502
http_503 http_504;
	}
}


2011/11/12 MagicBear <magicbearmo at gmail.com>:
> 2011/11/12 19:00:16 [alert] 7552#0: ignore long locked inactive cache
> entry 26b0312d67bd41ef132ce5b8a4445ffa, count:1
> 2011/11/12 19:02:17 [alert] 7552#0: ignore long locked inactive cache
> entry ac307ce9b33a01a04f4f17c187d9b11a, count:1
> 2011/11/12 19:02:45 [alert] 7552#0: ignore long locked inactive cache
> entry e5fa15e3f856238feb5e0b7128120e20, count:1
> 2011/11/12 19:03:59 [alert] 7552#0: ignore long locked inactive cache
> entry 1eb06fe015c489159f15b514bb333931, count:1
> 2011/11/12 19:04:46 [alert] 7552#0: ignore long locked inactive cache
> entry 5023f1eb7e74908ae75d6a7a57ac4dfd, count:2
> 2011/11/12 19:05:41 [alert] 7552#0: ignore long locked inactive cache
> entry 9fda125ea01601b6a32536afd2c59aa2, count:1
> 2011/11/12 19:06:02 [alert] 7547#0: worker process 7548 exited on
> signal 11 (core dumped)
>
>
>
> nginx: nginx version: nginx/1.1.7
> nginx: built by gcc 4.6.1 (Ubuntu/Linaro 4.6.1-9ubuntu3)
> nginx: TLS SNI support enabled
> nginx: configure arguments: --conf-path=/etc/nginx/nginx.conf
> --error-log-path=/var/log/nginx/error.log
> --pid-path=/var/run/nginx.pid --lock-path=/var/lock/nginx.lock
> --http-log-path=/var/log/nginx/access.log
> --http-client-body-temp-path=/var/lib/nginx/body
> --http-fastcgi-temp-path=/var/lib/nginx/fastcgi
> --http-proxy-temp-path=/var/lib/nginx/proxy
> --with-http_stub_status_module --without-select_module
> --without-http_uwsgi_module --without-http_auth_basic_module
> --without-http_geo_module --without-http_empty_gif_module
> --with-http_ssl_module --with-http_gzip_static_module
> --with-http_realip_module --add-module=../ngx_cache_purge-1.4
> --with-google_perftools_module
> --add-module=../chaoslawful-lua-nginx-module-71993f1
> --add-module=../nginx-accesskey-2.0.3/
>
>
>
>
> GNU gdb (Ubuntu/Linaro 7.3-0ubuntu2) 7.3-2011.08
> Copyright (C) 2011 Free Software Foundation, Inc.
> License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
> This is free software: you are free to change and redistribute it.
> There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
> and "show warranty" for details.
> This GDB was configured as "x86_64-linux-gnu".
> For bug reporting instructions, please see:
> <http://bugs.launchpad.net/gdb-linaro/>...
> "/var/www/ngx_coredump/core": not in executable format: File format
> not recognized
> "/usr/sbin/nginx" is not a core dump: File format not recognized
> (gdb) bt
> No stack.
> (gdb) ^CQuit
> (gdb) ^CQuit
> (gdb) quit
> root at cn-sd-wt:/var/www/ngx_coredump# gdb `which nginx` core
> GNU gdb (Ubuntu/Linaro 7.3-0ubuntu2) 7.3-2011.08
> Copyright (C) 2011 Free Software Foundation, Inc.
> License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
> This is free software: you are free to change and redistribute it.
> There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
> and "show warranty" for details.
> This GDB was configured as "x86_64-linux-gnu".
> For bug reporting instructions, please see:
> <http://bugs.launchpad.net/gdb-linaro/>...
> Reading symbols from /usr/sbin/nginx...done.
> [New LWP 7548]
>
> warning: Can't read pathname for load map: Input/output error.
> [Thread debugging using libthread_db enabled]
> Core was generated by `nginx:'.
> Program terminated with signal 11, Segmentation fault.
> #0  ngx_rbtree_min (sentinel=<optimized out>, node=0x0) at
> src/core/ngx_rbtree.h:75
> 75          while (node->left != sentinel) {
> (gdb) bt
> #0  ngx_rbtree_min (sentinel=<optimized out>, node=0x0) at
> src/core/ngx_rbtree.h:75
> #1  ngx_rbtree_delete (tree=0x7f59d00d7000, node=0x7f59d00db080) at
> src/core/ngx_rbtree.c:178
> #2  0x0000000000451a04 in ngx_http_file_cache_free (c=0x1c8d4f0,
> tf=0x1c8dfd0) at src/http/ngx_http_file_cache.c:956
> #3  0x0000000000447739 in ngx_http_upstream_process_request
> (r=0x21b8af0) at src/http/ngx_http_upstream.c:2738
> #4  0x000000000044781b in ngx_http_upstream_process_upstream
> (r=0x21b8af0, u=0x21e0c88) at src/http/ngx_http_upstream.c:2677
> #5  0x000000000044790c in ngx_http_upstream_handler (ev=<optimized
> out>) at src/http/ngx_http_upstream.c:935
> #6  0x0000000000425959 in ngx_event_expire_timers () at
> src/event/ngx_event_timer.c:149
> #7  0x0000000000425590 in ngx_process_events_and_timers
> (cycle=0x1a45c40) at src/event/ngx_event.c:261
> #8  0x000000000042b2d7 in ngx_worker_process_cycle (cycle=0x1a45c40,
> data=<optimized out>) at src/os/unix/ngx_process_cycle.c:801
> #9  0x0000000000429cd2 in ngx_spawn_process (cycle=0x1a45c40,
> proc=0x42b206 <ngx_worker_process_cycle>, data=0x0, name=0x47e0e5
> "worker process", respawn=-3) at src/os/unix/ngx_process.c:196
> #10 0x000000000042a986 in ngx_start_worker_processes (cycle=0x1a45c40,
> n=4, type=-3) at src/os/unix/ngx_process_cycle.c:360
> #11 0x000000000042b990 in ngx_master_process_cycle (cycle=0x1a45c40)
> at src/os/unix/ngx_process_cycle.c:136
> #12 0x00000000004119ea in main (argc=<optimized out>, argv=<optimized
> out>) at src/core/nginx.c:403
> (gdb) bt full
> #0  ngx_rbtree_min (sentinel=<optimized out>, node=0x0) at
> src/core/ngx_rbtree.h:75
> No locals.
> #1  ngx_rbtree_delete (tree=0x7f59d00d7000, node=0x7f59d00db080) at
> src/core/ngx_rbtree.c:178
>        red = <optimized out>
>        root = 0x7f59d00d7000
>        sentinel = 0x7f59d00d7018
>        subst = <optimized out>
>        temp = <optimized out>
>        w = <optimized out>
> #2  0x0000000000451a04 in ngx_http_file_cache_free (c=0x1c8d4f0,
> tf=0x1c8dfd0) at src/http/ngx_http_file_cache.c:956
>        cache = 0x1a97650
>        fcn = 0x7f59d00db080
> #3  0x0000000000447739 in ngx_http_upstream_process_request
> (r=0x21b8af0) at src/http/ngx_http_upstream.c:2738
>        del = <optimized out>
>        tf = <optimized out>
>        p = 0x21e1078
>        u = 0x21e0c88
> #4  0x000000000044781b in ngx_http_upstream_process_upstream
> (r=0x21b8af0, u=0x21e0c88) at src/http/ngx_http_upstream.c:2677
>        c = <optimized out>
> #5  0x000000000044790c in ngx_http_upstream_handler (ev=<optimized
> out>) at src/http/ngx_http_upstream.c:935
>        c = 0x7f59b9db70d0
>        r = <optimized out>
>        ctx = <optimized out>
>        u = <optimized out>
> #6  0x0000000000425959 in ngx_event_expire_timers () at
> src/event/ngx_event_timer.c:149
>        ev = 0x7f59ea790520
>        root = 0x7f59ea784da8
>        sentinel = 0x6a86e0
> #7  0x0000000000425590 in ngx_process_events_and_timers
> (cycle=0x1a45c40) at src/event/ngx_event.c:261
>        flags = 1
>        timer = <optimized out>
>        delta = 21
> #8  0x000000000042b2d7 in ngx_worker_process_cycle (cycle=0x1a45c40,
> data=<optimized out>) at src/os/unix/ngx_process_cycle.c:801
>        i = <optimized out>
>        c = <optimized out>
> #9  0x0000000000429cd2 in ngx_spawn_process (cycle=0x1a45c40,
> proc=0x42b206 <ngx_worker_process_cycle>, data=0x0, name=0x47e0e5
> "worker process", respawn=-3) at src/os/unix/ngx_process.c:196
>        on = 1
>        pid = 0
>        s = 0
> #10 0x000000000042a986 in ngx_start_worker_processes (cycle=0x1a45c40,
> n=4, type=-3) at src/os/unix/ngx_process_cycle.c:360
>        i = <optimized out>
>        ch = {command = 1, pid = 11, slot = 0, fd = 844191584}
> #11 0x000000000042b990 in ngx_master_process_cycle (cycle=0x1a45c40)
> at src/os/unix/ngx_process_cycle.c:136
>        title = 0x1bf78c6 "master process nginx"
>        p = <optimized out>
>        size = <optimized out>
>        i = <optimized out>
>        n = <optimized out>
>        sigio = <optimized out>
>        set = {__val = {0 <repeats 16 times>}}
>        itv = {it_interval = {tv_sec = 0, tv_usec = 0}, it_value =
> {tv_sec = 0, tv_usec = 5}}
>        live = <optimized out>
>        delay = <optimized out>
>        ls = <optimized out>
>        ccf = 0x1a46b50
> #12 0x00000000004119ea in main (argc=<optimized out>, argv=<optimized
> out>) at src/core/nginx.c:403
>        i = <optimized out>
>        log = 0x6a64a0
>        cycle = 0x1a45c40
>        init_cycle = {conf_ctx = 0x0, pool = 0x1a450d0, log =
> 0x6a64a0, new_log = {log_level = 0, file = 0x0, connection = 0,
> handler = 0, data = 0x0, action = 0x0}, files = 0x0, free_connections
> = 0x0, free_connection_n = 0,
>          reusable_connections_queue = {prev = 0x0, next = 0x0},
> listening = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0},
> pathes = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0},
> open_files = {last = 0x0,
>            part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0,
> nalloc = 0, pool = 0x0}, shared_memory = {last = 0x0, part = {elts =
> 0x0, nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0},
> connection_n = 0, files_n = 0,
>          connections = 0x0, read_events = 0x0, write_events = 0x0,
> old_cycle = 0x0, conf_file = {len = 21, data = 0x47a836
> "/etc/nginx/nginx.conf"}, conf_param = {len = 0, data = 0x0},
> conf_prefix = {len = 11,
>            data = 0x47a836 "/etc/nginx/nginx.conf"}, prefix = {len =
> 17, data = 0x47a824 "/usr/local/nginx/"}, lock_file = {len = 0, data =
> 0x0}, hostname = {len = 0, data = 0x0}}
>        ccf = <optimized out>
> (gdb)
>
>
> --
> MagicBear
>



-- 
MagicBear



More information about the nginx-devel mailing list