[PATCH] Add support for using sendfile when openssl support ktls

[ben ben ishay]

# HG changeset patch
# User ben ben ishay <benishay at mellanox.com>
# Date 1554896607 -10800
#      Wed Apr 10 14:43:27 2019 +0300
# Node ID 87938decdb98bf4a06ed18002a15156a5e8fbd67
# Parent  65074e13f1716e09c28d730586babad7930b7a98
Add support for using sendfile when openssl support ktls

when we need to transfer data between file and socket we prefer to use sendfile instead of write because we save the copy to a buffer.
the use of sendfile is possible in openssl only if it support ktls(the master of openssl support ktls) otherwise there is a copy of the data to userspace for encryption in any case (this paper explain this https://netdevconf.org/1.2/papers/ktls.pdf ).
the patch  change the flow when the request is to send data over ssl and also the nginx use openssl that support ktls, the new flow using the sendfile function that tcp use for send data (ngx_linux_sendfile_chain).
the performence with this patch applied was check with apib benchmark(https://github.com/apigee/apib), one machine run nginx and the other machine that connect back to back to the first one run apib with this comand: ./apib -c <num of connection> -d 30 https://<ip address>/<file name to send>.
the file size was 100K.

the result display  in this table , each value represnt average throughput in GBps of 10 runs.

num of connection   | regular nginx  | new nginx
	1		5		5.2
	2		7.5		8.5
	3		7.7		9

this result prove that this patch increase nginx performance and thus is useful.

diff -r 65074e13f171 -r 87938decdb98 auto/feature
--- a/auto/feature	Tue Mar 26 09:33:57 2019 +0300
+++ b/auto/feature	Wed Apr 10 14:43:27 2019 +0300
@@ -41,6 +41,10 @@
 
 ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS $ngx_feature_inc_path \
           -o $NGX_AUTOTEST $NGX_AUTOTEST.c $NGX_TEST_LD_OPT $ngx_feature_libs"
+if [ "$ngx_feature_name"  == "NGX_OPENSSL_KTLS" ];then
+        ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS $ngx_feature_inc_path -I$OPENSSL/include \
+          -o $NGX_AUTOTEST $NGX_AUTOTEST.c $NGX_TEST_LD_OPT $ngx_feature_libs"
+fi
 
 ngx_feature_inc_path=
 
diff -r 65074e13f171 -r 87938decdb98 auto/lib/openssl/conf
--- a/auto/lib/openssl/conf	Tue Mar 26 09:33:57 2019 +0300
+++ b/auto/lib/openssl/conf	Wed Apr 10 14:43:27 2019 +0300
@@ -140,3 +140,12 @@
     fi
 
 fi
+ngx_feature="OpenSSL library with KTLS"
+ngx_feature_name="NGX_OPENSSL_KTLS"
+ngx_feature_run=no
+ngx_feature_incs="#include \"openssl/bio.h\" "
+ngx_feature_path=
+ngx_feature_libs="-lssl -lcrypto $NGX_LIBDL $NGX_LIBPTHREAD"
+ngx_feature_test="BIO_get_ktls_send(NULL)"
+. auto/feature
+
diff -r 65074e13f171 -r 87938decdb98 src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c	Tue Mar 26 09:33:57 2019 +0300
+++ b/src/event/ngx_event_openssl.c	Wed Apr 10 14:43:27 2019 +0300
@@ -1528,6 +1528,9 @@
 #endif
 
     sc->connection = SSL_new(ssl->ctx);
+#if (NGX_OPENSSL_KTLS)
+    sc->ktls = 0;
+#endif
 
     if (sc->connection == NULL) {
         ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "SSL_new() failed");
@@ -1639,6 +1642,12 @@
         c->recv_chain = ngx_ssl_recv_chain;
         c->send_chain = ngx_ssl_send_chain;
 
+#if (NGX_OPENSSL_KTLS)
+       if(BIO_get_ktls_send(SSL_get_wbio(c->ssl->connection))){
+           c->ssl->ktls = 1;
+           c->send_chain = ngx_linux_sendfile_chain;
+	}
+#endif
 #ifndef SSL_OP_NO_RENEGOTIATION
 #if OPENSSL_VERSION_NUMBER < 0x10100000L
 #ifdef SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS
diff -r 65074e13f171 -r 87938decdb98 src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h	Tue Mar 26 09:33:57 2019 +0300
+++ b/src/event/ngx_event_openssl.h	Wed Apr 10 14:43:27 2019 +0300
@@ -99,6 +99,9 @@
     unsigned                    in_early:1;
     unsigned                    early_preread:1;
     unsigned                    write_blocked:1;
+#if (NGX_OPENSSL_KTLS)
+    unsigned                    ktls:1;
+#endif
 };
 
 
diff -r 65074e13f171 -r 87938decdb98 src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c	Tue Mar 26 09:33:57 2019 +0300
+++ b/src/http/ngx_http_request.c	Wed Apr 10 14:43:27 2019 +0300
@@ -604,9 +604,15 @@
     }
 
 #if (NGX_HTTP_SSL)
-    if (c->ssl) {
+#ifndef NGX_OPENSSL_KTLS
+    if (c->ssl){
         r->main_filter_need_in_memory = 1;
     }
+#else
+    if(!c->ssl->ktls && c->ssl){
+        r->main_filter_need_in_memory = 1;
+        }
+#endif
 #endif
 
     r->main = r;