Nginx KTLS hardware offloading not working
liwuliu
nginx-forum at forum.nginx.org
Mon Jun 13 23:57:26 UTC 2022
Hi Team,
I used Nginx to do 443:443 reverse proxy with Mellanox Connect6 DX
networking cards.
I can make KTLS work for Nginx, but cannot see KTLS offloading (inline
TLS @ MLX6) working.
Please help on what I missed?
Many thanks,
Liwu
-----------------
To utilize Openssh 3.0 and Nginx 1.21.1: I followed this instruction:
https://www.nginx.com/blog/improving-nginx-performance-with-kernel-tls/
To enable MLX6 inline TLS I followed this instruction:
https://docs.nvidia.com/networking/display/OFEDv521040/Kernel+Transport+Layer+Security+(kTLS)+Offloads
Here are further system information:
root at r57-8814:/boot# nginx -V
nginx version: nginx/1.21.4
built by gcc 11.2.0 (Ubuntu 11.2.0-19ubuntu1)
built with OpenSSL 3.0.0 7 sep 2021
TLS SNI support enabled
configure arguments: --with-debug --prefix=/usr/local
--conf-path=/usr/local/etc/nginx/nginx.conf
--error-log-path=/var/log/nginx/error.log
--http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid
--lock-path=/var/run/nginx.lock
--http-client-body-temp-path=/var/cache/nginx/client_temp
--http-proxy-temp-path=/var/cache/nginx/proxy_temp
--http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp
--http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp
--http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx
--with-compat --with-file-aio --with-threads --with-http_addition_module
--with-http_auth_request_module --with-http_dav_module
--with-http_flv_module --with-http_gunzip_module
--with-http_gzip_static_module --with-http_mp4_module
--with-http_random_index_module --with-http_realip_module
--with-http_secure_link_module --with-http_slice_module
--with-http_ssl_module --with-http_stub_status_module --with-http_sub_module
--with-http_v2_module --with-mail --with-mail_ssl_module --with-stream
--with-stream_realip_module --with-stream_ssl_module
--with-stream_ssl_preread_module --with-openssl=../openssl-3.0.0
--with-openssl-opt=enable-ktls --with-cc-opt='-g -O2
-fstack-protector-strong -Wformat -Werror=format-security
-Wp,-D_FORTIFY_SOURCE=2 -fPIC'
root at r57-8814:~# uname -a
Linux r57-8814 5.15.0-37-generic #39-Ubuntu SMP Wed Jun 1 19:16:45 UTC 2022
x86_64 x86_64 x86_64 GNU/Linux
root at r57-8814:~# ethtool -k enp202s0f0np0 |grep tls
tls-hw-tx-offload: on
tls-hw-rx-offload: on
tls-hw-record: off [fixed]
root at r57-8814:~# ethtool -k enp202s0f1np1 |grep tls
tls-hw-tx-offload: on
tls-hw-rx-offload: on
tls-hw-record: off [fixed]
root at r57-8814:~# lsmod |grep tls
tls 106496 77 mlx5_core
root at r57-8814:/boot# grep TLS config-5.15.0-37-generic
CONFIG_TLS=m
CONFIG_TLS_DEVICE=y
# CONFIG_TLS_TOE is not set
CONFIG_CHELSIO_TLS_DEVICE=m
CONFIG_MLX5_FPGA_TLS=y
CONFIG_MLX5_TLS=y
CONFIG_MLX5_EN_TLS=y
CONFIG_FB_TFT_TLS8204=m
root at r57-8814:/usr/local/etc/nginx# cat nginx.conf
#user nobody;
worker_processes 4;
worker_cpu_affinity 0001 0010 0100 1000;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request"
'
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
upstream backend {
server 1.1.2.2:443;
server 1.1.2.3:443;
server 1.1.2.4:443;
server 1.1.2.5:443;
server 1.1.2.6:443;
server 1.1.2.7:443;
server 1.1.2.8:443;
server 1.1.2.9:443;
server 1.1.2.10:443;
}
server {
listen 443 ssl;
ssl_certificate /usr/local/etc/nginx/cert.crt;
ssl_certificate_key /usr/local/etc/nginx/cert.key;
ssl_session_cache builtin:1000 shared:SSL:10m;
ssl_conf_command Options KTLS;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
ssl_ciphers
HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;
ssl_prefer_server_ciphers on;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
location / {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For
$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Fix the “It appears that your reverse proxy set up is
broken" error.
proxy_pass https://backend;
proxy_ssl_certificate /usr/local/etc/nginx/cert.crt;
proxy_ssl_certificate_key /usr/local/etc/nginx/cert.key;
proxy_ssl_trusted_certificate
/usr/local/etc/nginx/cert.crt;
proxy_ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
proxy_ssl_ciphers
HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;
}
}
Though the following stats suggest the inline-TLS is not triggered.
root at r57-8814:/boot# ethtool -S enp202s0f1np1 |grep tls
tx_tls_encrypted_packets: 0
tx_tls_encrypted_bytes: 0
tx_tls_ooo: 0
tx_tls_dump_packets: 0
tx_tls_dump_bytes: 0
tx_tls_resync_bytes: 0
tx_tls_skip_no_sync_data: 0
tx_tls_drop_no_sync_data: 0
tx_tls_drop_bypass_req: 0
rx_tls_decrypted_packets: 0
rx_tls_decrypted_bytes: 0
rx_tls_resync_req_pkt: 0
rx_tls_resync_req_start: 0
rx_tls_resync_req_end: 0
rx_tls_resync_req_skip: 0
rx_tls_resync_res_ok: 0
rx_tls_resync_res_retry: 0
rx_tls_resync_res_skip: 0
rx_tls_err: 0
tx_tls_ctx: 0
tx_tls_del: 0
rx_tls_ctx: 0
rx_tls_del: 0
root at r57-8814:/boot# ethtool -S enp202s0f0np0 |grep tls
tx_tls_encrypted_packets: 0
tx_tls_encrypted_bytes: 0
tx_tls_ooo: 0
tx_tls_dump_packets: 0
tx_tls_dump_bytes: 0
tx_tls_resync_bytes: 0
tx_tls_skip_no_sync_data: 0
tx_tls_drop_no_sync_data: 0
tx_tls_drop_bypass_req: 0
rx_tls_decrypted_packets: 0
rx_tls_decrypted_bytes: 0
rx_tls_resync_req_pkt: 0
rx_tls_resync_req_start: 0
rx_tls_resync_req_end: 0
rx_tls_resync_req_skip: 0
rx_tls_resync_res_ok: 0
rx_tls_resync_res_retry: 0
rx_tls_resync_res_skip: 0
rx_tls_err: 0
tx_tls_ctx: 0
tx_tls_del: 0
rx_tls_ctx: 0
rx_tls_del: 0
Posted at Nginx Forum: https://forum.nginx.org/read.php?2,294477,294477#msg-294477
More information about the nginx
mailing list