worker process exited on signal 11

mscheel nginx-forum at nginx.us
Tue Apr 17 13:44:13 UTC 2012 

Dear all,

error logs show this error (worker process exited on signal 11). The
error appears after ~1 hour, then with the half of the worker processes.
No problems with 1.0-versions of nginx. Problem existed in 1.1.18
already.

I'm using a custom rpm build with the following additional modules:
--add-module=/root/rpm/SOURCES/agentzh-headers-more-nginx-module-3580526
\
--add-module=/root/rpm/SOURCES/FRiCKLE-ngx_cache_purge-e34bd2c \
--add-module=/root/rpm/SOURCES/substitutions4nginx-read-only \

Here is the backtrace, any ideas?

Thank you.

2012/04/17 10:16:36 [alert] 20802#0: worker process 20808 exited on
signal 11 (core dumped)
2012/04/17 10:16:45 [alert] 20802#0: worker process 20805 exited on
signal 11 (core dumped)
2012/04/17 10:17:33 [alert] 20802#0: worker process 23616 exited on
signal 11 (core dumped)
2012/04/17 10:18:14 [alert] 20802#0: worker process 20806 exited on
signal 11 (core dumped)
2012/04/17 10:22:41 [alert] 20802#0: worker process 23892 exited on
signal 11 (core dumped)

dmesg:
nginx[20808]: segfault at 0000000000000020 rip 000000000042a47b rsp
00007fff79073f80 error 4
nginx[20805]: segfault at 0000000000000020 rip 000000000042a47b rsp
00007fff79073f80 error 4
nginx[23616]: segfault at 0000000000000020 rip 000000000042a47b rsp
00007fff79073ff0 error 4
nginx[20806]: segfault at 0000000000000020 rip 000000000042a47b rsp
00007fff79073f80 error 4
nginx[23892]: segfault at 0000000000000008 rip 000000000042cbe8 rsp
00007fff79073f20 error 6



gdb /usr/sbin/nginx /var/local/dumps/core
GNU gdb 6.6
Copyright (C) 2006 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you
are
welcome to change it and/or distribute copies of it under certain
conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for
details.
This GDB was configured as "x86_64-suse-linux"...
Using host libthread_db library "/lib64/libthread_db.so.1".
Reading symbols from /lib64/libpthread.so.0...done.
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libcrypt.so.1...done.
Loaded symbols for /lib64/libcrypt.so.1
Reading symbols from /usr/lib64/libpcre.so.0...done.
Loaded symbols for /usr/lib64/libpcre.so.0
Reading symbols from /usr/lib64/libssl.so.0.9.8...done.
Loaded symbols for /usr/lib64/libssl.so.0.9.8
Reading symbols from /usr/lib64/libcrypto.so.0.9.8...done.
Loaded symbols for /usr/lib64/libcrypto.so.0.9.8
Reading symbols from /lib64/libdl.so.2...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libz.so.1...done.
Loaded symbols for /lib64/libz.so.1
Reading symbols from /usr/lib64/libGeoIP.so.1...done.
Loaded symbols for /usr/lib64/libGeoIP.so.1
Reading symbols from
/usr/lib/perl5/5.8.8/x86_64-linux-thread-multi/CORE/libperl.so...done.
Loaded symbols for
/usr/lib/perl5/5.8.8/x86_64-linux-thread-multi/CORE/libperl.so
Reading symbols from /lib64/libm.so.6...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /lib64/libc.so.6...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from
/usr/lib/perl5/site_perl/5.8.8/x86_64-linux-thread-multi/auto/nginx/nginx.so...done.
Loaded symbols for
/usr/lib/perl5/site_perl/5.8.8/x86_64-linux-thread-multi/auto/nginx/nginx.so
Core was generated by `nginx: worker pr'.
Program terminated with signal 11, Segmentation fault.
#0  0x000000000042cbe8 in ngx_readv_chain (c=0x2b508a1d43d0,
chain=0x602348) at src/os/unix/ngx_readv_chain.c:195
195     src/os/unix/ngx_readv_chain.c: No such file or directory.
        in src/os/unix/ngx_readv_chain.c
(gdb) bt
#0  0x000000000042cbe8 in ngx_readv_chain (c=0x2b508a1d43d0,
chain=0x602348) at src/os/unix/ngx_readv_chain.c:195
#1  0x000000000042b133 in ngx_event_pipe (p=0x747170, do_write=<value
optimized out>) at src/event/ngx_event_pipe.c:271
#2  0x000000000045410e in ngx_http_upstream_process_upstream
(r=0x69ef00, u=0x746db8) at src/http/ngx_http_upstream.c:2641
#3  0x0000000000454269 in ngx_http_upstream_handler (ev=0x2b50cc7d9218)
at src/http/ngx_http_upstream.c:935
#4  0x0000000000428f64 in ngx_event_process_posted (cycle=0x5f6490,
posted=0x5d14a0) at src/event/ngx_event_posted.c:40
#5  0x000000000042fc36 in ngx_worker_process_cycle (cycle=0x5f6490,
data=<value optimized out>) at src/os/unix/ngx_process_cycle.c:806
#6  0x000000000042e14d in ngx_spawn_process (cycle=0x5f6490,
proc=0x42fb70 <ngx_worker_process_cycle>, data=0x0, name=0x49e05e
"worker process", respawn=5) at src/os/unix/ngx_process.c:198
#7  0x0000000000430c76 in ngx_master_process_cycle (cycle=0x5f6490) at
src/os/unix/ngx_process_cycle.c:622
#8  0x000000000041235d in main (argc=1, argv=<value optimized out>) at
src/core/nginx.c:410
(gdb) backtrace full
#0  0x000000000042cbe8 in ngx_readv_chain (c=0x2b508a1d43d0,
chain=0x602348) at src/os/unix/ngx_readv_chain.c:195
        prev = (u_char *) 0x0
        n = <value optimized out>
        size = 0
        err = <value optimized out>
        vec = {elts = 0x7fff79073f20, nelts = 0, size = 16, nalloc = 16,
pool = 0x5f57a0}
        rev = <value optimized out>
        iov = (struct iovec *) 0x0
        iovs = {{iov_base = 0x40000, iov_len = 140735223906344},
{iov_base = 0x69ef00, iov_len = 47624914552144}, {iov_base = 0x6024c0,
iov_len = 262144}, {iov_base = 0x1, iov_len = 4566241}, {
    iov_base = 0x40000, iov_len = 6942464}, {iov_base = 0x2b508a1d4550,
iov_len = 0}, {iov_base = 0x0, iov_len = 47624930312022}, {iov_base =
0x69b6e0, iov_len = 6942464}, {
    iov_base = 0x2b508a1d4550, iov_len = 6300864}, {iov_base = 0x40000,
iov_len = 1}, {iov_base = 0x602460, iov_len = 4367464}, {iov_base = 0x1,
iov_len = 2}, {iov_base = 0x0,
    iov_len = 47624930399933}, {iov_base = 0x40000, iov_len = 4309842},
{iov_base = 0x2b50cc80e2e8, iov_len = 4370439}, {iov_base = 0x2b,
iov_len = 2}}
#1  0x000000000042b133 in ngx_event_pipe (p=0x747170, do_write=<value
optimized out>) at src/event/ngx_event_pipe.c:271
        rc = <value optimized out>
        rev = <value optimized out>
        wev = <value optimized out>
#2  0x000000000045410e in ngx_http_upstream_process_upstream
(r=0x69ef00, u=0x746db8) at src/http/ngx_http_upstream.c:2641
        c = (ngx_connection_t *) 0x2b508a1d43d0
#3  0x0000000000454269 in ngx_http_upstream_handler (ev=0x2b50cc7d9218)
at src/http/ngx_http_upstream.c:935
        c = (ngx_connection_t *) 0x2b508a1d4550
        r = (ngx_http_request_t *) 0x69ef00
        u = (ngx_http_upstream_t *) 0x746db8
#4  0x0000000000428f64 in ngx_event_process_posted (cycle=0x5f6490,
posted=0x5d14a0) at src/event/ngx_event_posted.c:40
        ev = (ngx_event_t *) 0x2b50cc7d9218
#5  0x000000000042fc36 in ngx_worker_process_cycle (cycle=0x5f6490,
data=<value optimized out>) at src/os/unix/ngx_process_cycle.c:806
        i = 0
        c = <value optimized out>
#6  0x000000000042e14d in ngx_spawn_process (cycle=0x5f6490,
proc=0x42fb70 <ngx_worker_process_cycle>, data=0x0, name=0x49e05e
"worker process", respawn=5) at src/os/unix/ngx_process.c:198
        on = 1
        pid = 0
        s = 5
#7  0x0000000000430c76 in ngx_master_process_cycle (cycle=0x5f6490) at
src/os/unix/ngx_process_cycle.c:622
        title = <value optimized out>
        p = <value optimized out>
        size = 9
        i = <value optimized out>
        n = 432
        sigio = 0
        set = {__val = {0 <repeats 16 times>}}
        itv = {it_interval = {tv_sec = 0, tv_usec = 6251688}, it_value =
{tv_sec = 0, tv_usec = 47624916165168}}
        live = 1
        delay = 0
        ccf = (ngx_core_conf_t *) 0x5f74f0
#8  0x000000000041235d in main (argc=1, argv=<value optimized out>) at
src/core/nginx.c:410
        i = <value optimized out>
        log = (ngx_log_t *) 0x5ce6e0
        cycle = (ngx_cycle_t *) 0x5f6490
        init_cycle = {conf_ctx = 0x0, pool = 0x5f5440, log = 0x5ce6e0,
new_log = {log_level = 0, file = 0x0, connection = 0, handler = 0, data
= 0x0, action = 0x0}, files = 0x0,
  free_connections = 0x0, free_connection_n = 0,
reusable_connections_queue = {prev = 0x0, next = 0x0}, listening = {elts
= 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, pathes = {
    elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, open_files
= {last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0,
nalloc = 0, pool = 0x0}, shared_memory = {
    last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0,
nalloc = 0, pool = 0x0}, connection_n = 0, files_n = 0, connections =
0x0, read_events = 0x0, write_events = 0x0,
  old_cycle = 0x0, conf_file = {len = 21, data = 0x4998b3
"/etc/nginx/nginx.conf"}, conf_param = {len = 0, data = 0x0},
conf_prefix = {len = 11, data = 0x4998b3 "/etc/nginx/nginx.conf"},
  prefix = {len = 6, data = 0x4998ac "/usr//"}, lock_file = {len = 0,
data = 0x0}, hostname = {len = 0, data = 0x0}}
        ccf = (ngx_core_conf_t *) 0x5f74f0

Posted at Nginx Forum: http://forum.nginx.org/read.php?2,225420,225420#msg-225420

More information about the nginx mailing list