Nginx Container crash no logs

[Richard Dakin]

Hey all,

We have a new set up running large amounts of data through a container
nginx. This is crashing, without error. Forcing a reboot to recover daily
at the moment. We are getting nothing from nginx logs or the docker logs of
any use. Any suggestions to debugging this?

CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
9bc827d6ccd7 nginx:stable

Container is running on ubuntu 18.04.3 LTS OS.
The last thing we see in the logs is

18:15:36 [alert] 7#7: ignore long locked inactive cache entry
9f78089258be73e98f58abed986ddb8b, count:1
2019/11/13 18:25:36 [alert] 7#7: ignore long locked inactive cache entry
9f78089258be73e98f58abed986ddb8b, count:1
2019/11/13 18:35:36 [alert] 7#7: ignore long locked inactive cache entry
9f78089258be73e98f58abed986ddb8b, count:1

We're tried a lot of things such as down grading the nginx version as we
were using latest but this just prolonged the crash time. Small improvement
by a day
Any suggestions welcome

Nginx conf is below

Windows Terminal
worker_rlimit_nofile 30000;
events {}
http {
log_format compression '$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent" "$gzip_ratio"';
error_log /etc/nginx/error_log.log warn;
client_max_body_size 20m;
server_names_hash_bucket_size 512;
proxy_headers_hash_bucket_size 128;
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=content_cache:10m
max_size=10g use_temp_path=off;
upstream hub_node {
server hub-node:3000;
keepalive 16;
}
upstream hub_cms {
server hub-be:80;
keepalive 16;
}
upstream hub_analytics {
server hub-matomo:80;
keepalive 16;
}

server {
listen 443 default_server;
server_name _;
return 418;
}

server {
listen 443 ssl http2;
server_name digital-hub.bwi.dpn.gov.uk;
add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive";
location /sites/default/files/ {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
proxy_cache_valid 200 302 10m;
proxy_cache content_cache;
proxy_pass http://hub_cms/sites/default/files/;

}
location / {
access_log /var/log/nginx/access.log compression buffer=32k;
proxy_pass http://hub_node/;
}
}

server {
listen 443 ssl http2;
server_name analytics.digital-hub.bwi.dpn.gov.uk;
add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive";
location / {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://hub_analytics/;
}
}

server {
listen 443 ssl http2;
server_name content.digital-hub.bwi.dpn.gov.uk;
add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive";
location / {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
proxy_cache_valid 200 302 10m;
proxy_cache content_cache;
proxy_pass http://hub_cms/;
}
}

ssl_certificate /etc/letsencrypt/live/localhost/san.digital-hub.crt;
ssl_certificate_key /etc/letsencrypt/live/localhost/san.digital-hub.rsa;
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
ssl_session_tickets off;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers
ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-
SHA256:DHE-RSA-AES256-GCM-SHA384;
ssl_prefer_server_ciphers off;
# HSTS (ngx_http_headers_module is required) (63072000 seconds)
add_header Strict-Transport-Security "max-age=63072000" always;
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx/attachments/20191113/d8ec5973/attachment.htm>