[PATCH] when we need to transfer data between file and socket we prefer to use sendfile instead of write because we save the copy to a buffer

[ben ben ishay]

# HG changeset patch
# User ben ben ishay <benishay at mellanox.com>
# Date 1555572726 -10800
#      Thu Apr 18 10:32:06 2019 +0300
# Node ID bb4c564a9f1c5c721c192e6188967c19aabbc0b9
# Parent  a6e23e343081b79eb924da985a414909310aa7a3
when we need to transfer data between file and socket we prefer to use sendfile instead of write because we save the copy to a buffer.
the use of sendfile is possible in openssl only if it support ktls(the master of openssl support ktls) otherwise there is a copy of the data to userspace for encryption in any case (this paper explain this https://netdevconf.org/1.2/papers/ktls.pdf ).
the patch  change the flow when the request is to send data over ssl and also the nginx use openssl that support ktls, the new flow using the sendfile function that tcp use for send data (ngx_linux_sendfile_chain).
the performence with this patch applied was check with apib benchmark(this is the source https://github.com/apigee/apib),one machine run nginx and the other machine that connect back to back to the first one run apib with this comand: ./apib -c <num of connection> -d 30 https://<ip address>/<file name to send>.
the file size was 100K.

the result display  in this table , each value represnt average throughput in GBps of 10 runs.

num of connection   | regular nginx  | new nginx
 	1		5		5.2
 	2		7.5		8.5
 	3		7.7		9

this result prove that this patch increase nginx performance and thus is useful.

diff -r a6e23e343081 -r bb4c564a9f1c src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c	Tue Apr 09 16:00:30 2019 +0300
+++ b/src/event/ngx_event_openssl.c	Thu Apr 18 10:32:06 2019 +0300
@@ -1529,6 +1529,9 @@
 
     sc->connection = SSL_new(ssl->ctx);
 
+#ifdef  BIO_get_ktls_send
+    sc->ktls = 0;
+#endif
     if (sc->connection == NULL) {
         ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "SSL_new() failed");
         return NGX_ERROR;
@@ -1639,6 +1642,13 @@
         c->recv_chain = ngx_ssl_recv_chain;
         c->send_chain = ngx_ssl_send_chain;
 
+#if (NGX_LINUX)
+#ifdef BIO_get_ktls_send
+        if(BIO_get_ktls_send(SSL_get_wbio(c->ssl->connection)))
+            c->ssl->ktls = 1;
+	    c->send_chain = ngx_send_chain;
+#endif
+#endif
 #ifndef SSL_OP_NO_RENEGOTIATION
 #if OPENSSL_VERSION_NUMBER < 0x10100000L
 #ifdef SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS
diff -r a6e23e343081 -r bb4c564a9f1c src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h	Tue Apr 09 16:00:30 2019 +0300
+++ b/src/event/ngx_event_openssl.h	Thu Apr 18 10:32:06 2019 +0300
@@ -99,6 +99,9 @@
     unsigned                    in_early:1;
     unsigned                    early_preread:1;
     unsigned                    write_blocked:1;
+#ifdef BIO_get_ktls_send
+    unsigned                    ktls:1;
+#endif
 };
 
 
diff -r a6e23e343081 -r bb4c564a9f1c src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c	Tue Apr 09 16:00:30 2019 +0300
+++ b/src/http/ngx_http_request.c	Thu Apr 18 10:32:06 2019 +0300
@@ -604,9 +604,15 @@
     }
 
 #if (NGX_HTTP_SSL)
-    if (c->ssl) {
+#ifndef BIO_get_ktls_send
+    if (c->ssl){
         r->main_filter_need_in_memory = 1;
     }
+#else
+    if(c->ssl && !c->ssl->ktls){
+        r->main_filter_need_in_memory = 1;
+    }
+#endif
 #endif
 
     r->main = r;
diff -r a6e23e343081 -r bb4c564a9f1c src/os/unix/ngx_linux_sendfile_chain.c
--- a/src/os/unix/ngx_linux_sendfile_chain.c	Tue Apr 09 16:00:30 2019 +0300
+++ b/src/os/unix/ngx_linux_sendfile_chain.c	Thu Apr 18 10:32:06 2019 +0300
@@ -256,7 +256,15 @@
     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
                    "sendfile: @%O %uz", file->file_pos, size);
 
+#if (NGX_HTTP_SSL)
+    if (c->ssl) {
+        n = SSL_sendfile(c->ssl->connection, file->file->fd, offset, size, 0);
+    } else {
+        n = sendfile(c->fd, file->file->fd, &offset, size);
+    }
+#else
     n = sendfile(c->fd, file->file->fd, &offset, size);
+#endif
 
     if (n == -1) {
         err = ngx_errno;