worker process exited on signal 11

Wed Apr 18 05:06:25 UTC 2012

Hello Maxim,

it appears only on two productive servers. Compiling without these
modules is not possible at the moment but i've tried several times with
each module deactivated (and without any of these three).

Full config is huge. I'm using a php-fpm backend with keepalive on and
one upstream server with nginx and php-fpm.

  http {
	include mime.types;
	include gzip.conf;
	default_type  application/octet-stream;
	keepalive_timeout  240;
	keepalive_requests 2000;
	open_file_cache max=10000 inactive=3m; 
	open_file_cache_valid    3m; 
	open_file_cache_min_uses 2;
	open_file_cache_errors   off;
	index  wartung.html index.php index.html index.htm;
	recursive_error_pages on;

	large_client_header_buffers 8 256k;
	client_max_body_size 1024m;
	client_body_buffer_size 128K;
	client_header_buffer_size 10k;
	client_header_timeout 120;
	client_body_timeout 120;
	fastcgi_read_timeout 240;

	set_real_ip_from x.x.x.0/24;
	real_ip_header X-Forwarded-For;

	#nginx 1.0
		#limit_zone spiders $binary_remote_addr 10m;
	#nginx 1.1
		limit_conn_zone $binary_remote_addr zone=spiders:10m;

	limit_req_zone  $binary_remote_addr  zone=spider_req:10m   rate=5r/s;
	limit_conn_log_level info;

	upstream be_server_nginx {
		server IP:80;
		keepalive 300;
	}
	}

	...

	location @be_nginx {
	proxy_pass http://be_server_nginx;
	proxy_set_header Connection "";	proxy_http_version 1.1;
	include /etc/nginx/proxy.conf;

output of gdb:

(gdb) fr 0
#0  0x000000000042cbe8 in ngx_readv_chain (c=0x2b508a1d43d0,
chain=0x602348) at src/os/unix/ngx_readv_chain.c:195
195     in src/os/unix/ngx_readv_chain.c

(gdb) p chain
$1 = (ngx_chain_t *) 0x602348

(gdb) p *chain
$2 = {buf = 0x602460, next = 0x752628}

(gdb) fr 1
#1  0x000000000042b133 in ngx_event_pipe (p=0x747170, do_write=<value
optimized out>) at src/event/ngx_event_pipe.c:271
271     src/event/ngx_event_pipe.c: No such file or directory.
        in src/event/ngx_event_pipe.c

(gdb) p *p
$3 = {upstream = 0x2b508a1d43d0, downstream = 0x2b508a1d4550,
free_raw_bufs = 0x0, in = 0x6024d0, last_in = 0x7523a0, out = 0x0, free
= 0x602450, busy = 0x0,
  input_filter = 0x47bc30 <ngx_http_fastcgi_input_filter>, input_ctx =
0x69ef00, output_filter = 0x43ca40 <ngx_http_output_filter>, output_ctx
= 0x69ef00, read = 0, cacheable = 0,
  single_buf = 0, free_bufs = 1, upstream_done = 0, upstream_error = 0,
upstream_eof = 0, upstream_blocked = 0, downstream_done = 0,
downstream_error = 0, cyclic_temp_file = 0, allocated = 2,
  bufs = {num = 10, size = 262144}, tag = 0x5c8480, busy_size = 262144,
read_length = 378731, length = 1, max_temp_file_size = 0,
temp_file_write_size = 524288, read_timeout = 180000,
  send_timeout = 60000, send_lowat = 0, pool = 0x69b6e0, log = 0x6c5380,
preread_bufs = 0x0, preread_size = 129547, buf_to_file = 0x0, temp_file
= 0x602358, num = 6}

(gdb) fr 2
#2  0x000000000045410e in ngx_http_upstream_process_upstream
(r=0x69ef00, u=0x746db8) at src/http/ngx_http_upstream.c:2641
2641    src/http/ngx_http_upstream.c: No such file or directory.
        in src/http/ngx_http_upstream.c

(gdb) p *r
$4 = {signature = 1347703880, connection = 0x2b508a1d4550, ctx =
0x69baf0, main_conf = 0x5f7910, srv_conf = 0x617f48, loc_conf =
0x640e00,
  read_event_handler = 0x454510
<ngx_http_upstream_rd_check_broken_connection>, write_event_handler =
0x454b50 <ngx_http_upstream_process_downstream>, cache = 0x0, upstream =
0x746db8,
  upstream_states = 0x747890, pool = 0x69b6e0, header_in = 0x6c5450,
headers_in = {headers = {last = 0x69ef70, part = {elts = 0x69c008, nelts
= 11, next = 0x0}, size = 48, nalloc = 20,
      pool = 0x69b6e0}, host = 0x69c0f8, connection = 0x0,
if_modified_since = 0x0, if_unmodified_since = 0x0, user_agent =
0x69c098, referer = 0x69c038, content_length = 0x0,
    content_type = 0x0, range = 0x0, if_range = 0x0, transfer_encoding =
0x0, expect = 0x0, accept_encoding = 0x69c0c8, via = 0x69c188,
authorization = 0x0, keep_alive = 0x0,
    x_forwarded_for = 0x69c1b8, x_real_ip = 0x0, depth = 0x0,
destination = 0x0, overwrite = 0x0, date = 0x0, user = {len = 0, data =
0x0}, passwd = {len = 0, data = 0x0}, cookies = {
      elts = 0x69c3c8, nelts = 1, size = 8, nalloc = 2, pool =
0x69b6e0}, server = {len = 22, data = 0x74eadb "domain.com"},
content_length_n = -1, keep_alive_n = -1,
    connection_type = 2, msie = 1, msie6 = 0, opera = 0, gecko = 0,
chrome = 0, safari = 0, konqueror = 0}, headers_out = {headers = {last =
0x69f0c8, part = {elts = 0x69b730, nelts = 5,
        next = 0x0}, size = 48, nalloc = 20, pool = 0x69b6e0}, status =
200, status_line = {len = 6, data = 0x4a2662 "200 OK"}, server =
0x69b790, date = 0x0, content_length = 0x0,
    content_encoding = 0x69b7f0, location = 0x0, refresh = 0x0,
last_modified = 0x0, content_range = 0x0, accept_ranges = 0x0,
www_authenticate = 0x0, expires = 0x0, etag = 0x0,
    override_charset = 0x0, content_type_len = 9, content_type = {len =
24, data = 0x747a0d "text/html; charset=utf-8"}, charset = {len = 5,
data = 0x747a20 "utf-8"},
    content_type_lowcase = 0x602078 "text/html", content_type_hash =
101824002410317, cache_control = {elts = 0x6016c8, nelts = 1, size = 8,
nalloc = 2, pool = 0x69b6e0}, content_length_n = -1,
    date_time = 0, last_modified_time = -1}, request_body = 0x747270,
lingering_time = 0, start_sec = 1334650957, start_msec = 91, method = 2,
http_version = 1001, request_line = {len = 50,
    data = 0x74e8c0 "GET /nc/pdf-generation.html?pageID=776404
HTTP/1.1\r\nAccept"}, uri = {len = 10, data = 0x746d62 "/index.php"},
args = {len = 13, data = 0x746a8f "pageID=776404"}, exten = {
    len = 3, data = 0x746d69 "php"}, unparsed_uri = {len = 37, data =
0x74e8c4 "/nc/pdf-generation.html?pageID=776404 HTTP/1.1\r\nAccept"},
method_name = {len = 3,
    data = 0x74e8c0 "GET /nc/pdf-generation.html?pageID=776404
HTTP/1.1\r\nAccept"}, http_protocol = {len = 8, data = 0x74e8ea
"HTTP/1.1\r\nAccept"}, out = 0x7523b8, main = 0x69ef00,
  parent = 0x0, postponed = 0x0, post_subrequest = 0x0, posted_requests
= 0x0, virtual_names = 0x744098, phase_handler = 13, content_handler =
0x47b940 <ngx_http_fastcgi_handler>,
  access_code = 0, variables = 0x69bc98, ncaptures = 0, captures =
0x69c580, captures_data = 0x746a84 "/index.php?pageID=776404",
limit_rate = 0, header_size = 339, request_length = 785,
  err_status = 0, http_connection = 0x6c53d8, log_handler = 0x442e60
<ngx_http_log_error_handler>, cleanup = 0x7478f0, subrequests = 201,
count = 1, blocked = 0, aio = 0, http_state = 2,
  complex_uri = 0, quoted_uri = 0, plus_in_uri = 0, space_in_uri = 0,
invalid_header = 0, add_uri_to_alias = 0, valid_location = 1,
valid_unparsed_uri = 0, uri_changed = 0, uri_changes = 10,
  request_body_in_single_buf = 0, request_body_in_file_only = 0,
request_body_in_persistent_file = 0, request_body_in_clean_file = 0,
request_body_file_group_access = 0,
  request_body_file_log_level = 5, subrequest_in_memory = 0, waited = 0,
cached = 0, gzip_tested = 1, gzip_ok = 1, gzip_vary = 1, proxy = 0,
bypass_cache = 0, no_cache = 0, limit_conn_set = 1,
  limit_req_set = 1, pipeline = 0, plain_http = 0, chunked = 1,
header_only = 0, keepalive = 1, lingering_close = 0, discard_body = 0,
internal = 1, error_page = 0, ignore_content_encoding = 0,
  filter_finalize = 0, post_action = 0, request_complete = 0,
request_output = 1, header_sent = 1, expect_tested = 0, root_tested = 0,
done = 0, logged = 0, buffered = 0,
  main_filter_need_in_memory = 1, filter_need_in_memory = 1,
filter_need_temporary = 0, allow_ranges = 0, stat_reading = 0,
stat_writing = 1, state = 0, header_hash = 6740241951503570450,
  lowcase_index = 13, lowcase_header = "cache-controlsitiont", '\0'
<repeats 11 times>,
  header_name_start = 0x2b50cc8430b3
"^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×ÿÐßãßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^"...,
  header_name_end = 0x2b50cc8430a8
"û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×ÿÐßãßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û"...,
  header_start = 0x2b50cc8430aa
"uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×ÿÐßãßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯u"...,
  header_end = 0x2b50cc8430b3
"^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×ÿÐßãßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^"...,
uri_start = 0x74e8c4 "/nc/pdf-generation.html?pageID=776404
HTTP/1.1\r\nAccept", uri_end = 0x74e8e9 " HTTP/1.1\r\nAccept",
  uri_ext = 0x74e8d7 "html?pageID=776404 HTTP/1.1\r\nAccept", args_start
= 0x74e8dc "pageID=776404 HTTP/1.1\r\nAccept",
  request_start = 0x74e8c0 "GET /nc/pdf-generation.html?pageID=776404
HTTP/1.1\r\nAccept", request_end = 0x74e8f2 "\r\nAccept",
  method_end = 0x74e8c2 "T /nc/pdf-generation.html?pageID=776404
HTTP/1.1\r\nAccept", schema_start = 0x0, schema_end = 0x0, host_start =
0x0, host_end = 0x0, port_start = 0x0, port_end = 0x0,
  http_minor = 1, http_major = 1}

(gdb) p *u
$5 = {read_event_handler = 0x454090
<ngx_http_upstream_process_upstream>, write_event_handler = 0x452980
<ngx_http_upstream_dummy_handler>, peer = {connection = 0x2b508a1d43d0,
    sockaddr = 0x617ea8, socklen = 110, name = 0x68a6b0, tries = 1, get
= 0x487370 <ngx_http_upstream_get_keepalive_peer>, free = 0x4875b0
<ngx_http_upstream_free_keepalive_peer>,
    data = 0x747908, set_session = 0x486e80
<ngx_http_upstream_keepalive_set_session>, save_session = 0x486e90
<ngx_http_upstream_keepalive_save_session>, local = 0x0, rcvbuf = 0,
    log = 0x6c5380, cached = 1, log_error = 1}, pipe = 0x747170,
request_bufs = 0x747880, output = {buf = 0x0, in = 0x0, free = 0x0, busy
= 0x0, sendfile = 0, directio = 0, unaligned = 0,
    need_in_memory = 0, need_in_temp = 0, alignment = 512, pool =
0x69b6e0, allocated = 0, bufs = {num = 1, size = 131072}, tag =
0x5c8480, output_filter = 0x414db0 <ngx_chain_writer>,
    filter_ctx = 0x746eb0}, writer = {out = 0x0, last = 0x746eb0,
connection = 0x2b508a1d43d0, pool = 0x69b6e0, limit = 0}, conf =
0x6416a8, headers_in = {headers = {last = 0x746ee8, part = {
        elts = 0x601520, nelts = 4, next = 0x0}, size = 48, nalloc = 8,
pool = 0x69b6e0}, status_n = 200, status_line = {len = 6, data =
0x4a2662 "200 OK"}, status = 0x0, date = 0x0,
    server = 0x0, connection = 0x0, expires = 0x0, etag = 0x0,
x_accel_expires = 0x0, x_accel_redirect = 0x0, x_accel_limit_rate = 0x0,
content_type = 0x601580, content_length = 0x601520,
    last_modified = 0x0, location = 0x0, accept_ranges = 0x0,
www_authenticate = 0x0, transfer_encoding = 0x0, content_encoding = 0x0,
content_length_n = 355230, cache_control = {
      elts = 0x747a38, nelts = 1, size = 8, nalloc = 2, pool =
0x69b6e0}, connection_close = 0, chunked = 0}, resolved = 0x0, buffer =
{
    pos = 0x2b50cc843010
"¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î"...,
    last = 0x2b50cc843010
"¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î"...,
file_pos = 0, file_last = 0,
    start = 0x2b50cc843010
"¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î½×½û¯uï~ëÝ{ßº÷^÷î"...,
end = 0x2b50cc883010 "", tag = 0x5c8480, file = 0x0, shadow = 0x0,
temporary = 1, memory = 0, mmap = 0, recycled = 1, in_file = 0, flush =
0, sync = 0,
    last_buf = 0, last_in_chain = 0, last_shadow = 0, temp_file = 0, num
= 2}, length = 355230, out_bufs = 0x0, busy_bufs = 0x0, free_bufs =
0x0,
  input_filter_init = 0x47af80 <ngx_http_fastcgi_input_filter_init>,
input_filter = 0, input_filter_ctx = 0x0, create_key = 0x47c100
<ngx_http_fastcgi_create_key>,
  create_request = 0x47c3a0 <ngx_http_fastcgi_create_request>,
reinit_request = 0x47af50 <ngx_http_fastcgi_reinit_request>,
process_header = 0x47cf20 <ngx_http_fastcgi_process_header>,
  abort_request = 0x47b560 <ngx_http_fastcgi_abort_request>,
finalize_request = 0x47b530 <ngx_http_fastcgi_finalize_request>,
rewrite_redirect = 0, rewrite_cookie = 0, timeout = 0,
  state = 0x7478b8, method = {len = 0, data = 0x0}, schema = {len = 10,
data = 0x4a8c69 "fastcgi://"}, uri = {len = 0, data = 0x0}, cleanup =
0x7478f0, store = 0, cacheable = 0, accel = 0,
  ssl = 0, cache_status = 0, buffering = 1, keepalive = 0, request_sent
= 1, header_sent = 1}

Posted at Nginx Forum: http://forum.nginx.org/read.php?2,225420,225457#msg-225457