Nginx most connections in FIN_WAIT2 state

[RasmithaM]

We are using Nginx for outbound connectivity to client , 

I see all the requests are going to FIN_WAIT2 state , even server sending us
the ACK. 

the fin_timeout is set to 60 sec , but we observed that the process
continues to stay in FIN_WAIT2 even after 60sec. 
Is this kernel issue / Nginc issue ?
netstat -tan | awk '{print $6}' | sort | uniq -c
   1793 CLOSE_WAIT
     40 ESTABLISHED
   6398 FIN_WAIT2
      1 Foreign
     22 LISTEN
    152 TIME_WAIT
      1 established)

This is filling up the number of sockets finally have to restart Nginx to
release the FIN_WAIT2 processes. 

Nginx configuration : 
egress-service-meshproxy.conf: |
server {

  listen 9080;

  server_name www.services.com;

  proxy_set_header X-Real-IP $remote_addr;
  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  proxy_set_header X-NginX-Proxy true;

  proxy_cache_bypass $http_upgrade;
  proxy_redirect off;

  proxy_ssl_protocols TLSv1.2 TLSv1.3;
  proxy_ssl_ciphers   HIGH:!aNULL:!MD5;

  proxy_read_timeout 10s;
  proxy_connect_timeout 10s;

  # this doesn't seem to work well of "on" -- 502 upstream drop from on
reused connections
  proxy_http_version 1.1;
  proxy_set_header Connection "";
  proxy_ssl_session_reuse off;

  #proxy_ssl_name off;
  proxy_ssl_server_name on;

  proxy_ssl_verify on;
  proxy_ssl_verify_depth 3;


  location / {


    proxy_ssl_certificate /deployment/secrets/egress-service-prod/tls.crt;
    proxy_ssl_certificate_key
/deployment/secrets/egress-service-prod/tls.key;
    #proxy_ssl_trusted_certificate
/deployment/secrets/egress-service-prod/ca.crt;
    proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;


    proxy_pass https://www.services.com:443;
  }
}

nginx-server-default.conf: |+
server {

  listen 9080 default_server;
  listen [::]:9080 default_server;

  root /usr/share/nginx/html;

  index index.html;

  # Proxy everything we know about to static content
  location /api/v1/irp/health {
    add_header Content-Type text/plain;
    return 200 '{ "status": "OK" }';
  }
  location /api/v1/irp/actuator/health {
    add_header Content-Type text/plain;
    return 200 '{ "status": "OK" }';
  }
  location / {
    add_header Content-Type text/plain;
    return 200 '{ "status": "OK, no content here, use the  services hostname
to access SSL reverse proxy!" }';
  }

}

nginx.conf: |+

pcre_jit on;
user  nginx;
worker_processes  1;
error_log  /var/log/nginx/error.log debug;
pid        /var/run/nginx.pid;
events {
    worker_connections        2048;
    accept_mutex              off;
    multi_accept              off;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format main '{"time": "$time_local","status":
"$status","request_time": $request_time, "host": "$http_host", "port":
"$server_port", "request_uri": "$uri",
"x_et_request_id":"$http_x_et_request_id","x_et_response_code":
"$upstream_http_x_et_response_code"}';      
    access_log  /var/log/nginx/access.log  main;
    error_log  /var/log/nginx/error.log;
    sendfile                  on;
    tcp_nopush                on;
    tcp_nodelay               on;
    client_max_body_size      10m;
    keepalive_timeout         60;
    #ssl_prefer_server_ciphers on;
    #use                      epoll;
    gzip  on;

    include /deployment/config/nginx-server-default.conf;
    include /deployment/config/egress-service-meshproxy-*.conf;
}

template-nginx-server.conf: |-

server {

  listen 9080;
  
  server_name ${MESH_HOSTNAME};

  proxy_set_header X-Real-IP $remote_addr;
  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  proxy_set_header X-NginX-Proxy true;  

  proxy_cache_bypass $http_upgrade;
  proxy_redirect off;

  proxy_ssl_protocols TLSv1.2 TLSv1.3;
  proxy_ssl_ciphers   HIGH:!aNULL:!MD5;

  proxy_read_timeout 10s;
  proxy_connect_timeout 10s;  

  # this doesn't seem to work well of "on" -- 502 upstream drop from  on
reused connections
  proxy_http_version 1.1;
  proxy_set_header Connection "";  
  proxy_ssl_session_reuse off;

  #proxy_ssl_name off;
  proxy_ssl_server_name on;
  proxy_ssl_verify on;
  proxy_ssl_verify_depth 3;

  location / {

    proxy_ssl_certificate /deployment/secrets/payaas-ipccpaas-com/tls.crt;
    proxy_ssl_certificate_key
/deployment/secrets/payaas-ipccpaas-com/tls.key;
    #proxy_ssl_trusted_certificate
/deployment/secrets/payaas-ipccpaas-com/ca.crt;
    proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
    proxy_pass https://${MESH_HOSTNAME};
  }
}

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,294880,294880#msg-294880