Nginx crash signal 6
wizard1024
nginx-forum на forum.nginx.org
Ср Июл 12 10:26:29 UTC 2017
Здравствуйте.
Worker нгинкса падает при обращении к сайту. В браузере - пустой ответ.
nginx -V:
nginx version: nginx/1.10.2
built by gcc 4.4.7 20120313 (Red Hat 4.4.7-17) (GCC)
built with OpenSSL 1.0.1e-fips 11 Feb 2013
TLS SNI support enabled
configure arguments: --prefix=/usr/share/nginx --sbin-path=/usr/sbin/nginx
--modules-path=/usr/lib64/nginx/modules --conf-path=/etc/nginx/nginx.conf
--error-log-path=/var/log/nginx/error.log
--http-log-path=/var/log/nginx/access.log
--http-client-body-temp-path=/var/lib/nginx/tmp/client_body
--http-proxy-temp-path=/var/lib/nginx/tmp/proxy
--http-fastcgi-temp-path=/var/lib/nginx/tmp/fastcgi
--http-uwsgi-temp-path=/var/lib/nginx/tmp/uwsgi
--http-scgi-temp-path=/var/lib/nginx/tmp/scgi --pid-path=/var/run/nginx.pid
--lock-path=/var/lock/subsys/nginx --user=nginx --group=nginx
--with-file-aio --with-ipv6 --with-http_ssl_module --with-http_v2_module
--with-http_realip_module --with-http_addition_module
--with-http_xslt_module=dynamic --with-http_image_filter_module=dynamic
--with-http_geoip_module=dynamic --with-http_sub_module
--with-http_dav_module --with-http_flv_module --with-http_mp4_module
--with-http_gunzip_module --with-http_gzip_static_module
--with-http_random_index_module --with-http_secure_link_module
--with-http_degradation_module --with-http_slice_module
--with-http_stub_status_module --with-http_perl_module=dynamic
--with-mail=dynamic --with-mail_ssl_module --with-pcre --with-pcre-jit
--with-stream=dynamic --with-stream_ssl_module --with-debug
--with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions
-fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic'
--with-ld-opt=' -Wl,-E'
uname -a:
Linux xxx-1.xxx.local 2.6.32-696.el6.x86_64 #1 SMP Tue Mar 21 19:29:05 UTC
2017 x86_64 x86_64 x86_64 GNU/Linux
В syslog:
Jul 12 03:45:37 xxx-1 kernel: nginx[662]: segfault at 7ffd4d0a1ff0 ip
0000000000459510 sp 00007ffd4d0a2008 error 6 in nginx[400000+dc000]
Core dump:
#0 ngx_http_get_indexed_variable (r=0x2149ce0, index=15) at
src/http/ngx_http_variables.c:491
v = <value optimized out>
cmcf = <value optimized out>
#1 0x000000000045c8a1 in ngx_http_script_copy_var_len_code (e=<value
optimized out>) at src/http/ngx_http_script.c:813
value = <value optimized out>
code = <value optimized out>
#2 0x000000000045d41d in ngx_http_complex_value (r=0x2149ce0,
val=0x2218128, value=0x7ffd4d0a20b0) at src/http/ngx_http_script.c:83
len = <value optimized out>
code = <value optimized out>
lcode = <value optimized out>
e = {ip = 0x2218180 "", pos = 0x0, sp = 0x0, buf = {len = 0, data =
0x0}, line = {len = 0, data = 0x0}, args = 0x0, flushed = 1, skip = 0, quote
= 0, is_args = 0, log = 0, status = 0,
request = 0x2149ce0}
#3 0x000000000048f084 in ngx_http_map_variable (r=0x2149ce0, v=0x214a908,
data=35750136) at src/http/modules/ngx_http_map_module.c:119
map = 0x22180f8
val = {len = 0, data = 0x0}
value = <value optimized out>
#4 0x0000000000459588 in ngx_http_get_indexed_variable (r=0x2149ce0,
index=<value optimized out>) at src/http/ngx_http_variables.c:509
v = <value optimized out>
cmcf = <value optimized out>
#5 0x000000000048f16c in ngx_http_map_variable (r=0x2149ce0, v=0x214a908,
data=35750136) at src/http/modules/ngx_http_map_module.c:134
map = 0x22180f8
val = {len = 0, data = 0x214ac88 "200 OK"}
value = <value optimized out>
#6 0x0000000000459588 in ngx_http_get_indexed_variable (r=0x2149ce0,
index=<value optimized out>) at src/http/ngx_http_variables.c:509
v = <value optimized out>
cmcf = <value optimized out>
#7 0x000000000048f16c in ngx_http_map_variable (r=0x2149ce0, v=0x214a908,
data=35750136) at src/http/modules/ngx_http_map_module.c:134
map = 0x22180f8
val = {len = 0, data = 0x214ac88 "200 OK"}
value = <value optimized out>
#8 0x0000000000459588 in ngx_http_get_indexed_variable (r=0x2149ce0,
index=<value optimized out>) at src/http/ngx_http_variables.c:509
v = <value optimized out>
cmcf = <value optimized out>
#9 0x000000000048f16c in ngx_http_map_variable (r=0x2149ce0, v=0x214a908,
data=35750136) at src/http/modules/ngx_http_map_module.c:134
map = 0x22180f8
val = {len = 0, data = 0x214ac88 "200 OK"}
value = <value optimized out>
#10 0x0000000000459588 in ngx_http_get_indexed_variable (r=0x2149ce0,
index=<value optimized out>) at src/http/ngx_http_variables.c:509
v = <value optimized out>
cmcf = <value optimized out>
...пропущено повторяющееся...
#262044 0x0000000000459588 in ngx_http_get_indexed_variable (r=0x2149ce0,
index=<value optimized out>) at src/http/ngx_http_variables.c:509
v = <value optimized out>
cmcf = <value optimized out>
#262045 0x000000000048f16c in ngx_http_map_variable (r=0x2149ce0,
v=0x214a908, data=35750136) at src/http/modules/ngx_http_map_module.c:134
map = 0x22180f8
val = {len = 0, data = 0x214ac88 "200 OK"}
value = <value optimized out>
#262046 0x0000000000459588 in ngx_http_get_indexed_variable (r=0x2149ce0,
index=<value optimized out>) at src/http/ngx_http_variables.c:509
v = <value optimized out>
cmcf = <value optimized out>
#262047 0x000000000048f16c in ngx_http_map_variable (r=0x2149ce0,
v=0x214a908, data=35750136) at src/http/modules/ngx_http_map_module.c:134
map = 0x22180f8
val = {len = 0, data = 0x214ac88 "200 OK"}
value = <value optimized out>
#262048 0x0000000000459588 in ngx_http_get_indexed_variable (r=0x2149ce0,
index=<value optimized out>) at src/http/ngx_http_variables.c:509
v = <value optimized out>
cmcf = <value optimized out>
#262049 0x000000000045c8a1 in ngx_http_script_copy_var_len_code (e=<value
optimized out>) at src/http/ngx_http_script.c:813
value = <value optimized out>
code = <value optimized out>
#262050 0x0000000000496f9d in ngx_http_proxy_create_request (r=0x2149ce0) at
src/http/modules/ngx_http_proxy_module.c:1247
len = 48
uri_len = <value optimized out>
loc_len = <value optimized out>
body_len = <value optimized out>
escape = <value optimized out>
b = <value optimized out>
method = {len = 3, data = 0x2114df0 "GET / HTTP/1.1\r\nHost"}
i = <value optimized out>
unparsed_uri = <value optimized out>
cl = <value optimized out>
body = <value optimized out>
part = <value optimized out>
header = <value optimized out>
u = 0x2270e70
ctx = 0x214aac0
code = <value optimized out>
headers = 0x2006078
e = {ip = 0x1 <Address 0x1 out of bounds>, pos = 0x3c18376b1c
"err.c", sp = 0x3c185e18c8, buf = {len = 258103742681, data = 0x1 <Address
0x1 out of bounds>}, line = {len = 258104322844,
data = 0x3c185e18c8 ""}, args = 0x3c1826a556
"D\211\350H\213\\$\030H\213l$
L\213d$(L\213l$0H\203\304\070\303fffff.\017\037\204", flushed = 0, skip = 0,
quote = 0, is_args = 0,
log = 0, status = 469, request = 0x0}
le = {ip = 0x22c5d10 "@\273E", pos = 0x0, sp = 0x0, buf = {len = 0,
data = 0x0}, line = {len = 0, data = 0x0}, args = 0x0, flushed = 1, skip =
0, quote = 0, is_args = 0, log = 0,
status = 0, request = 0x2149ce0}
plcf = 0x2005e80
lcode = <value optimized out>
#262051 0x0000000000464e98 in ngx_http_upstream_init_request (r=0x2149ce0)
at src/http/ngx_http_upstream.c:586
host = <value optimized out>
i = <value optimized out>
ctx = <value optimized out>
temp = {next = 0x7ffd4dca1050, resolver = 0x214a888, node =
0x2149ce0, ident = 34561088, state = 34908360, name = {len = 4771864, data =
0x21de820 "\002"}, service = {len = 257698037776,
data = 0x68 <Address 0x68 out of bounds>}, valid = 3864, naddrs
= 34905312, addrs = 0x21807c8, addr = {sockaddr = 0x20f5db8, socklen =
1305088160, name = {len = 140725908541744,
---Type <return> to continue, or q <return> to quit---
data = 0x45c7f6
"E\213\004$A\201\340\377\377\377\017N\215\004"}, priority = 23992, weight =
527}, sin = {sin_family = 40160, sin_port = 532, sin_addr = {s_addr = 0},
sin_zero = "0\021\312M\375\177\000"}, count = 4576381, nsrvs =
34561608, srvs = 0x214aab9, handler = 0, data = 0x1, timeout = 34908856,
quick = 4426590, recursion = 1,
event = 0x214aab9}
cln = <value optimized out>
u = 0x2270e70
clcf = <value optimized out>
uscf = <value optimized out>
uscfp = <value optimized out>
umcf = <value optimized out>
#262052 0x0000000000458b23 in ngx_http_read_client_request_body
(r=0x2149ce0, post_handler=0x4658c0 <ngx_http_upstream_init>) at
src/http/ngx_http_request_body.c:147
preread = <value optimized out>
size = <value optimized out>
rc = <value optimized out>
b = <value optimized out>
out = {buf = 0x2149ce0, next = 0x118}
rb = 0x2271278
#262053 0x0000000000496bed in ngx_http_proxy_handler (r=0x2149ce0) at
src/http/modules/ngx_http_proxy_module.c:914
rc = <value optimized out>
u = 0x2270e70
ctx = 0x214aac0
plcf = 0x2005e80
pmcf = <value optimized out>
#262054 0x0000000000448654 in ngx_http_core_content_phase (r=0x2149ce0,
ph=0x205f810) at src/http/ngx_http_core_module.c:1370
root = 4486661
rc = <value optimized out>
path = {len = 34905232, data = 0x0}
#262055 0x000000000044449d in ngx_http_core_run_phases (r=0x2149ce0) at
src/http/ngx_http_core_module.c:847
rc = <value optimized out>
ph = 0x205f6f0
cmcf = <value optimized out>
#262056 0x0000000000450861 in ngx_http_process_request (r=0x2149ce0) at
src/http/ngx_http_request.c:1910
c = 0x22cd990
#262057 0x00000000004516fc in ngx_http_process_request_line (rev=0x23048a0)
at src/http/ngx_http_request.c:1022
n = <value optimized out>
rc = <value optimized out>
rv = <value optimized out>
host = {len = 36718832, data = 0x400 <Address 0x400 out of bounds>}
c = 0x22cd990
r = 0x2149ce0
#262058 0x000000000042faf6 in ngx_event_process_posted (cycle=0x229b710,
posted=0x7042f0) at src/event/ngx_event_posted.c:33
q = 0x23048f0
ev = 0x23048a0
#262059 0x0000000000437045 in ngx_worker_process_cycle (cycle=0x229b710,
data=<value optimized out>) at src/os/unix/ngx_process_cycle.c:753
worker = <value optimized out>
#262060 0x00000000004355a4 in ngx_spawn_process (cycle=0x229b710,
proc=0x437000 <ngx_worker_process_cycle>, data=0x2, name=0x4b2c3e "worker
process", respawn=2) at src/os/unix/ngx_process.c:198
on = 1
pid = 0
s = <value optimized out>
#262061 0x0000000000437f22 in ngx_reap_children (cycle=0x229b710) at
src/os/unix/ngx_process_cycle.c:621
i = <value optimized out>
live = <value optimized out>
---Type <return> to continue, or q <return> to quit---
n = <value optimized out>
ch = {command = 2, pid = 31644, slot = 2, fd = -1}
ccf = <value optimized out>
#262062 ngx_master_process_cycle (cycle=0x229b710) at
src/os/unix/ngx_process_cycle.c:174
title = <value optimized out>
p = <value optimized out>
size = <value optimized out>
i = <value optimized out>
n = <value optimized out>
sigio = 0
set = {__val = {0 <repeats 16 times>}}
itv = {it_interval = {tv_sec = 0, tv_usec = 33116984}, it_value =
{tv_sec = 0, tv_usec = 0}}
live = <value optimized out>
delay = 0
ls = <value optimized out>
ccf = 0x229cc80
#262063 0x00000000004120cf in main (argc=<value optimized out>, argv=<value
optimized out>) at src/core/nginx.c:367
b = <value optimized out>
log = <value optimized out>
i = <value optimized out>
cycle = 0x1f95320
init_cycle = {conf_ctx = 0x0, pool = 0x1f94d70, log = 0x6f1020,
new_log = {log_level = 0, file = 0x0, connection = 0, disk_full_time = 0,
handler = 0, data = 0x0, writer = 0, wdata = 0x0,
action = 0x0, next = 0x0}, log_use_stderr = 0, files = 0x0,
free_connections = 0x0, free_connection_n = 0, modules = 0x0, modules_n = 0,
modules_used = 0,
reusable_connections_queue = {prev = 0x0, next = 0x0}, listening =
{elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, paths = {elts =
0x0, nelts = 0, size = 0, nalloc = 0,
pool = 0x0}, config_dump = {elts = 0x0, nelts = 0, size = 0,
nalloc = 0, pool = 0x0}, open_files = {last = 0x0, part = {elts = 0x0, nelts
= 0, next = 0x0}, size = 0, nalloc = 0,
pool = 0x0}, shared_memory = {last = 0x0, part = {elts = 0x0,
nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0}, connection_n = 0,
files_n = 0, connections = 0x0,
read_events = 0x0, write_events = 0x0, old_cycle = 0x0, conf_file
= {len = 21, data = 0x7ffd4dca1f7d "ss"}, conf_param = {len = 0, data =
0x0}, conf_prefix = {len = 11,
data = 0x7ffd4dca1f7d "ss"}, prefix = {len = 17, data = 0x4ad97e
"/usr/share/nginx/"}, lock_file = {len = 0, data = 0x0}, hostname = {len =
0, data = 0x0}}
cd = <value optimized out>
ccf = 0x1f96ee8
nginx.conf:
user nginx;
worker_processes 12;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
working_directory /tmp/cores/;
worker_rlimit_core 500M;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request"
'
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"
"$ssl_protocol/$ssl_cipher"';
log_format main_post '$remote_addr - $remote_user [$time_local]
"$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for" '
'"$http_host" "$request_time"
"$upstream_response_time" "$upstream_addr" "$request_body"
"$ssl_protocol/$ssl_cipher"';
log_format main_post-partner '$remote_addr - $remote_user
[$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for" '
'"$http_host" "$request_body"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
# Limits and Whitelist
geo $mywhitelist {
default 1;
#Good IPs
100.100.100.100 0;
111.111.111.111 0;
222.222.222.222 0;
}
map $mywhitelist $req_limit_zone {
1 $binary_remote_addr;
0 "";
}
map $http_x_real_ip $realip {
'' $remote_addr;
default $http_x_real_ip;
}
# No version on nginx
server_tokens off;
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 10m;
include /etc/nginx/sites/*;
}
В файле описания одного из сайтов встречается такой подозрительный момент
(уже закоментирован):
limit_req_zone $req_limit_zone zone=req-xxx:10m rate=25r/s;
#map $http_x_real_ip $realip {
# '' $realip;
# default $http_x_real_ip;
#}
upstream upstream-xxx.xxx {
server 127.0.0.1:1111;
server 192.168.1.2:1111;
server 192.168.2.2:1111 backup;
}
upstream upstream-xxx-1.xxx.xxx {
server 127.0.0.1:8173;
}
upstream upstream-xxx-2.xxx.xxx {
server 192.168.1.2:1111;
server 192.168.2.2:1111 backup;
}
server {
listen 153.120.184.201:80;
listen 153.120.184.203:80;
server_name xxx.xxx www.xxx.xxx;
location / {
return 301 https://www.xxx.xxx$request_uri;
}
}
server {
listen 200.200.200.200:443;
listen 200.200.200.203:443;
server_name xxx.xxx;
ssl on;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers
ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-RC4-SHA:ECDHE-RSA-RC4-SHA:ECDH-ECDSA-RC4-SHA:ECDH-RSA-RC4-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:HIGH:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!CBC:!EDH:!kEDH:!PSK:!SRP:!kECDH;
ssl_prefer_server_ciphers on;
ssl_verify_client off;
ssl_certificate /etc/nginx/ssl/www.xxx.xxx.crt;
ssl_certificate_key /etc/nginx/ssl/www.xxx.xxx.key;
access_log /var/log/nginx/www.xxx.xxx_acess.log main_post;
access_log syslog:server=192.168.0.100:11514 main_post-partner;
return 301 https://www.xxx.xxx$request_uri;
}
server {
listen 200.200.200.200:443;
listen 200.200.200.203:443;
server_name www.xxx.xxx;
ssl on;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers
ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-RC4-SHA:ECDHE-RSA-RC4-SHA:ECDH-ECDSA-RC4-SHA:ECDH-RSA-RC4-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:HIGH:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!CBC:!EDH:!kEDH:!PSK:!SRP:!kECDH;
ssl_prefer_server_ciphers on;
ssl_verify_client off;
ssl_certificate /etc/nginx/ssl/www.xxx.xxx.crt;
ssl_certificate_key /etc/nginx/ssl/www.xxx.xxx.key;
port_in_redirect on;
access_log /var/log/nginx/www.xxx.xxx_acess.log main_post;
access_log syslog:server=192.168.0.100:11514 main_post-partner;
error_log /var/log/nginx/www.xxx.xxx_error.log;
# Limit request
limit_req zone=req-xxx burst=25;
limit_req_log_level error;
#
location / {
proxy_pass http://127.0.0.1:3080;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $realip;
proxy_set_header X-Forwarded-For
$proxy_add_x_forwarded_for;
proxy_redirect https://$host/
https://$host:$server_port/;
proxy_set_header X-Forwarded-Proto https;
proxy_connect_timeout 300;
proxy_read_timeout 300;
proxy_send_timeout 300;
send_timeout 300;
client_max_body_size 40m;
}
...skiped...
}
# Transparent proxy
server {
listen localhost:3080;
server_name www.xxx.xxx;
access_log /var/log/nginx/www.xxx.xxx_acess.log main_post;
error_log /var/log/nginx/www.xxx.xxx_error.log;
#Main Backend
location / {
proxy_pass http://upstream-xxx.xxx;
proxy_set_header Host $http_host;
client_max_body_size 40M;
proxy_connect_timeout 300;
proxy_read_timeout 300;
proxy_send_timeout 300;
send_timeout 300;
}
location =/status-node1 {
rewrite /status-node1 /status break;
proxy_pass http://upstream-xxx-1.xxx.xxx;
}
location =/status-node2 {
rewrite /status-node2 /status break;
proxy_pass http://upstream-xxx-2.xxx.xxx;
}
}
# LAN answers
server {
listen 192.168.0.200:443;
server_name www.xxx.xxx;
ssl on;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers
ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:HIGH:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!CBC:!EDH:!kEDH:!PSK:!SRP:!kECDH:!RC4;
ssl_prefer_server_ciphers on;
ssl_verify_client off;
ssl_certificate /etc/nginx/ssl/www.xxx.xxx.crt;
ssl_certificate_key /etc/nginx/ssl/www.xxx.xxx.key;
port_in_redirect on;
access_log /var/log/nginx/answer.xxx.xxx_access.log main_post;
error_log /var/log/nginx/answer.xxx.xxx_error.log;
location / {
proxy_pass http://upstream-xxx.xxx;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $realip;
proxy_set_header X-Forwarded-For
$proxy_add_x_forwarded_for;
proxy_redirect https://$host/
https://$host:$server_port/;
proxy_set_header X-Forwarded-Proto https;
proxy_connect_timeout 300;
proxy_read_timeout 300;
proxy_send_timeout 300;
send_timeout 300;
client_max_body_size 40m;
}
}
После комментирования подозрительного момента - падения прекратились.
Воспроизвести ошибку на стенде не получилось. Проявляется не всегда.
Posted at Nginx Forum: https://forum.nginx.org/read.php?21,275444,275444#msg-275444
Подробная информация о списке рассылки nginx-ru