[PATCH v2] Add preadv2 support with RWF_NOWAIT flag

[Vadim Fedorenko]

# HG changeset patch
# User Vadim Fedorenko <vfedorenko at yandex-team.ru>
# Date 1515713238 -10800
#      Fri Jan 12 02:27:18 2018 +0300
# Node ID fbf6a421212b291cbacfcfc503173c0168449165
# Parent  93abb5a855d6534f0356882f45be49f8c6a95a8b
Add preadv2 support with RWF_NOWAIT flag

Introduction of thread pools is really good thing, but it adds
overhead to reading files which are already in page cache in linux.
With preadv2 (introduced in Linux 4.6) and RWF_NOWAIT flag (introduced
in Linux 4.14) we can eliminate this overhead. Needs glibc >= 2.26

This is v2 patch with code style fixes. Feature renamed to
NGX_HAVE_PREADV2_NOWAIT, call to preadv2() moved to ngx_thread_read(),
that's why it became simpler.

diff -r 93abb5a855d6 -r fbf6a421212b auto/unix
--- a/auto/unix Thu Jan 11 21:43:49 2018 +0300
+++ b/auto/unix Fri Jan 12 02:27:18 2018 +0300
@@ -727,6 +727,23 @@
 . auto/feature


+# preadv2() was introduced in Linux 4.6, glibc 2.26
+# RWF_NOWAIT flag was introduced in Linux 4.14
+
+ngx_feature="preadv2()"
+ngx_feature_name="NGX_HAVE_PREADV2_NOWAIT"
+ngx_feature_run=no
+ngx_feature_incs='#include <sys/uio.h>'
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="char buf[1]; struct iovec vec[1]; ssize_t n;
+                  vec[0].iov_base = buf;
+                  vec[0].iov_len = 1;
+                  n = preadv2(0, vec, 1, 0, RWF_NOWAIT);
+                  if (n == -1) return 1"
+. auto/feature
+
+
 ngx_feature="sys_nerr"
 ngx_feature_name="NGX_SYS_NERR"
 ngx_feature_run=value
diff -r 93abb5a855d6 -r fbf6a421212b src/os/unix/ngx_files.c
--- a/src/os/unix/ngx_files.c   Thu Jan 11 21:43:49 2018 +0300
+++ b/src/os/unix/ngx_files.c   Fri Jan 12 02:27:18 2018 +0300
@@ -26,6 +26,61 @@

 #endif

+#if (NGX_THREADS) && (NGX_HAVE_PREADV2_NOWAIT)
+
+ngx_uint_t  ngx_preadv2_nowait = 1;
+
+
+ssize_t
+ngx_preadv2_file(ngx_file_t *file, u_char *buf, size_t size, off_t offset)
+{
+    ssize_t  n;
+    struct iovec   iovs[1];
+
+    if (!ngx_preadv2_nowait) {
+        return NGX_AGAIN;
+    }
+
+    iovs[0].iov_base = buf;
+    iovs[0].iov_len = size;
+
+    n = preadv2(file->fd, iovs, 1, offset, RWF_NOWAIT);
+
+    if (n == -1) { /* let's analyze the return code */
+        switch (ngx_errno) {
+            case EAGAIN:
+                ngx_log_debug(NGX_LOG_DEBUG_CORE, file->log, 0,
+                              "preadv2() will block on \"%s\"",
+                              file->name.data);
+                return NGX_AGAIN;
+            case EINVAL:
+                /* Most possible case - not supported RWF_NOWAIT */
+                ngx_log_error(NGX_LOG_ERR, file->log, ngx_errno,
+                              "preadv2() \"%s\" failed RWF_NOWAIT",
+                              file->name.data);
+                ngx_preadv2_nowait = 0;
+                return NGX_AGAIN;
+            default:
+                return NGX_AGAIN;
+
+        }
+    }
+
+    /* Check if we read partial file */
+    if (((size_t)n < size) && (n < file->info.st_size)) {
+        /* blocked on partial read */
+        ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0,
+                       "preadv2() blocked partial on \"%s\" "
+                       "with read size %uz", file->name.data, n);
+        return NGX_AGAIN;
+    }
+
+    file->offset += n;
+
+    return n;
+}
+#endif
+

 ssize_t
 ngx_read_file(ngx_file_t *file, u_char *buf, size_t size, off_t offset)
@@ -97,6 +152,9 @@
 {
     ngx_thread_task_t      *task;
     ngx_thread_file_ctx_t  *ctx;
+#if (NGX_HAVE_PREADV2_NOWAIT)
+    ssize_t  n;
+#endif

     ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0,
                    "thread read: %d, %p, %uz, %O",
@@ -105,6 +163,15 @@
     task = file->thread_task;

     if (task == NULL) {
+#if (NGX_HAVE_PREADV2_NOWAIT)
+        n = ngx_preadv2_file(file, buf, size, offset);
+        if (n != NGX_AGAIN) {
+            ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0,
+                           "preadv2 non blocking: \"%s\" - %uz",
+                           file->name.data, n);
+            return n;
+        }
+#endif
         task = ngx_thread_task_alloc(pool, sizeof(ngx_thread_file_ctx_t));
         if (task == NULL) {
             return NGX_ERROR;
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20180112/cfb70c41/attachment.html>