cache: move open to thread pool

Roman Arutyunyan arut at nginx.com
Tue Aug 3 08:51:49 UTC 2021


Hi,

On Thu, Nov 01, 2018 at 07:08:10PM +0300, Maxim Konovalov wrote:
> On 01/11/2018 15:30, Roman Arutyunyan wrote:
> > Hi,
> > 
> > On Thu, Oct 04, 2018 at 12:32:26PM +0300, Roman Arutyunyan wrote:
> >> Hi,
> >>
> >> On Thu, Sep 13, 2018 at 09:10:23PM +0300, Maxim Dounin wrote:
> >>> Hello!
> >>>
> >>> On Tue, Sep 04, 2018 at 04:58:05PM -0700, Ka-Hing Cheung via nginx-devel wrote:
> >>>
> >>>> On Mon, Sep 3, 2018 at 9:09 AM, Maxim Dounin <mdounin at mdounin.ru> wrote:
> >> [..]
> >>
> >> Here's another approach to thread open.  This time it's 4 patches:
> >>
> >> - #1 a small open file cache refactoring
> >> - #2 thread open in open file cache
> >> - #3 thread open in http static module
> >> - #4 thread open in http file cache
> > The next iteration of the work.
> > Only 3 patches this time.
> 
> Testing and feedbacks are welcome.

An updated patchset which applies to the current nginx source code.

This work is still kept as a separate patchset since we have not got enough
feedback from potential users.  If you're interested in this feature, please
try these patches and report back to us.

-- 
Roman Arutyunyan
-------------- next part --------------
# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1540900849 -10800
#      Tue Oct 30 15:00:49 2018 +0300
# Node ID f386594b298d1442e1bfe1add9cdf3c6cf26917b
# Parent  1ebd78df4ce7262967c5dadce7bac454c4086896
Threaded open support in open file cache.

diff --git a/src/core/ngx_file.h b/src/core/ngx_file.h
--- a/src/core/ngx_file.h
+++ b/src/core/ngx_file.h
@@ -17,6 +17,7 @@ struct ngx_file_s {
     ngx_fd_t                   fd;
     ngx_str_t                  name;
     ngx_file_info_t            info;
+    ngx_err_t                  err;
 
     off_t                      offset;
     off_t                      sys_offset;
diff --git a/src/core/ngx_open_file_cache.c b/src/core/ngx_open_file_cache.c
--- a/src/core/ngx_open_file_cache.c
+++ b/src/core/ngx_open_file_cache.c
@@ -22,6 +22,8 @@
 
 
 static void ngx_open_file_cache_cleanup(void *data);
+static ngx_int_t ngx_open_uncached_file(ngx_str_t *name,
+    ngx_open_file_info_t *of, ngx_pool_t *pool);
 #if (NGX_HAVE_OPENAT)
 static ngx_fd_t ngx_openat_file_owner(ngx_fd_t at_fd, const u_char *name,
     ngx_int_t mode, ngx_int_t create, ngx_int_t access, ngx_log_t *log);
@@ -147,54 +149,15 @@ ngx_open_cached_file(ngx_open_file_cache
     time_t                          now;
     uint32_t                        hash;
     ngx_int_t                       rc;
-    ngx_file_info_t                 fi;
     ngx_pool_cleanup_t             *cln;
     ngx_cached_open_file_t         *file;
-    ngx_pool_cleanup_file_t        *clnf;
     ngx_open_file_cache_cleanup_t  *ofcln;
 
     of->fd = NGX_INVALID_FILE;
     of->err = 0;
 
     if (cache == NULL) {
-
-        if (of->test_only) {
-
-            if (ngx_file_info_wrapper(name, of, &fi, pool->log)
-                == NGX_FILE_ERROR)
-            {
-                return NGX_ERROR;
-            }
-
-            of->uniq = ngx_file_uniq(&fi);
-            of->mtime = ngx_file_mtime(&fi);
-            of->size = ngx_file_size(&fi);
-            of->fs_size = ngx_file_fs_size(&fi);
-            of->is_dir = ngx_is_dir(&fi);
-            of->is_file = ngx_is_file(&fi);
-            of->is_link = ngx_is_link(&fi);
-            of->is_exec = ngx_is_exec(&fi);
-
-            return NGX_OK;
-        }
-
-        cln = ngx_pool_cleanup_add(pool, sizeof(ngx_pool_cleanup_file_t));
-        if (cln == NULL) {
-            return NGX_ERROR;
-        }
-
-        rc = ngx_open_and_stat_file(name, of, pool->log);
-
-        if (rc == NGX_OK && !of->is_dir) {
-            cln->handler = ngx_pool_cleanup_file;
-            clnf = cln->data;
-
-            clnf->fd = of->fd;
-            clnf->name = name->data;
-            clnf->log = pool->log;
-        }
-
-        return rc;
+        return ngx_open_uncached_file(name, of, pool);
     }
 
     cln = ngx_pool_cleanup_add(pool, sizeof(ngx_open_file_cache_cleanup_t));
@@ -486,6 +449,163 @@ failed:
 }
 
 
+static ngx_int_t
+ngx_open_uncached_file(ngx_str_t *name, ngx_open_file_info_t *of,
+    ngx_pool_t *pool)
+{
+    ngx_fd_t                  fd;
+    ngx_int_t                 mode, create, access;
+    ngx_file_info_t           fi;
+    ngx_pool_cleanup_t       *cln;
+    ngx_pool_cleanup_file_t  *clnf;
+
+    if (of->test_only || of->test_dir) {
+
+        if (ngx_file_info_wrapper(name, of, &fi, pool->log) == NGX_FILE_ERROR) {
+            return NGX_ERROR;
+        }
+
+        if (of->test_only || ngx_is_dir(&fi)) {
+            goto done;
+        }
+    }
+
+    if (!of->log) {
+
+        /*
+         * Use non-blocking open() not to hang on FIFO files, etc.
+         * This flag has no effect on a regular files.
+         */
+
+        mode = NGX_FILE_RDONLY|NGX_FILE_NONBLOCK;
+        create = NGX_FILE_OPEN;
+        access = 0;
+
+    } else {
+        mode = NGX_FILE_APPEND;
+        create = NGX_FILE_CREATE_OR_OPEN;
+        access = NGX_FILE_DEFAULT_ACCESS;
+    }
+
+#if (NGX_THREADS)
+
+    if (of->thread_handler
+        && of->disable_symlinks == NGX_DISABLE_SYMLINKS_OFF)
+    {
+        ngx_int_t   rc;
+        ngx_file_t  file;
+
+        ngx_memzero(&file, sizeof(ngx_file_t));
+
+        file.log = pool->log;
+        file.fd = NGX_INVALID_FILE;
+        file.thread_handler = of->thread_handler;
+        file.thread_ctx = of->thread_ctx;
+        file.thread_task = of->thread_task;
+
+        rc = ngx_thread_open(&file, name->data, mode, create, access, pool);
+
+        if (rc == NGX_AGAIN) {
+            of->thread_task = file.thread_task;
+            return NGX_AGAIN;
+        }
+
+        if (rc != NGX_OK) {
+            of->err = file.err;
+            of->failed = ngx_open_file_n;
+            return NGX_ERROR;
+        }
+
+        fd = file.fd;
+
+    } else {
+        fd = ngx_open_file_wrapper(name, of, mode, create, access, pool->log);
+        if (fd == NGX_INVALID_FILE) {
+            return NGX_ERROR;
+        }
+    }
+
+#else
+    fd = ngx_open_file_wrapper(name, of, mode, create, access, pool->log);
+    if (fd == NGX_INVALID_FILE) {
+        return NGX_ERROR;
+    }
+#endif
+
+    if (ngx_fd_info(fd, &fi) == NGX_FILE_ERROR) {
+        ngx_log_error(NGX_LOG_CRIT, pool->log, ngx_errno,
+                      ngx_fd_info_n " \"%V\" failed", name);
+
+        if (ngx_close_file(fd) == NGX_FILE_ERROR) {
+            ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
+                          ngx_close_file_n " \"%V\" failed", name);
+        }
+
+        return NGX_ERROR;
+    }
+
+    if (ngx_is_dir(&fi)) {
+        if (ngx_close_file(fd) == NGX_FILE_ERROR) {
+            ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
+                          ngx_close_file_n " \"%V\" failed", name);
+        }
+
+    } else {
+        of->fd = fd;
+
+        if (of->read_ahead && ngx_file_size(&fi) > NGX_MIN_READ_AHEAD) {
+            if (ngx_read_ahead(fd, of->read_ahead) == NGX_ERROR) {
+                ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
+                              ngx_read_ahead_n " \"%V\" failed", name);
+            }
+        }
+
+        if (of->directio <= ngx_file_size(&fi)) {
+            if (ngx_directio_on(fd) == NGX_FILE_ERROR) {
+                ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
+                              ngx_directio_on_n " \"%V\" failed", name);
+
+            } else {
+                of->is_directio = 1;
+            }
+        }
+    }
+
+done:
+
+    of->uniq = ngx_file_uniq(&fi);
+    of->mtime = ngx_file_mtime(&fi);
+    of->size = ngx_file_size(&fi);
+    of->fs_size = ngx_file_fs_size(&fi);
+    of->is_dir = ngx_is_dir(&fi);
+    of->is_file = ngx_is_file(&fi);
+    of->is_link = ngx_is_link(&fi);
+    of->is_exec = ngx_is_exec(&fi);
+
+    if (of->fd != NGX_INVALID_FILE) {
+        cln = ngx_pool_cleanup_add(pool, sizeof(ngx_pool_cleanup_file_t));
+
+        if (cln == NULL) {
+            if (ngx_close_file(of->fd) == NGX_FILE_ERROR) {
+                ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
+                              ngx_close_file_n " \"%s\" failed", name->data);
+            }
+
+            return NGX_ERROR;
+        }
+
+        cln->handler = ngx_pool_cleanup_file;
+        clnf = cln->data;
+
+        clnf->fd = of->fd;
+        clnf->name = name->data;
+        clnf->log = pool->log;
+    }
+
+    return NGX_OK;
+}
+
+
 #if (NGX_HAVE_OPENAT)
 
 static ngx_fd_t
diff --git a/src/core/ngx_open_file_cache.h b/src/core/ngx_open_file_cache.h
--- a/src/core/ngx_open_file_cache.h
+++ b/src/core/ngx_open_file_cache.h
@@ -32,6 +32,13 @@ typedef struct {
 
     ngx_uint_t               min_uses;
 
+#if (NGX_THREADS || NGX_COMPAT)
+    ngx_int_t              (*thread_handler)(ngx_thread_task_t *task,
+                                             ngx_file_t *file);
+    void                    *thread_ctx;
+    ngx_thread_task_t       *thread_task;
+#endif
+
 #if (NGX_HAVE_OPENAT)
     size_t                   disable_symlinks_from;
     unsigned                 disable_symlinks:2;
diff --git a/src/os/unix/ngx_files.c b/src/os/unix/ngx_files.c
--- a/src/os/unix/ngx_files.c
+++ b/src/os/unix/ngx_files.c
@@ -11,6 +11,7 @@
 
 #if (NGX_THREADS)
 #include <ngx_thread_pool.h>
+static void ngx_thread_open_handler(void *data, ngx_log_t *log);
 static void ngx_thread_read_handler(void *data, ngx_log_t *log);
 static void ngx_thread_write_chain_to_file_handler(void *data, ngx_log_t *log);
 #endif
@@ -77,20 +78,112 @@ ngx_read_file(ngx_file_t *file, u_char *
 
 #if (NGX_THREADS)
 
+typedef enum {
+    NGX_THREAD_FILE_OPEN = 1,
+    NGX_THREAD_FILE_READ,
+    NGX_THREAD_FILE_WRITE
+} ngx_thread_file_op_e;
+
+
 typedef struct {
     ngx_fd_t       fd;
-    ngx_uint_t     write;   /* unsigned  write:1; */
+    u_char        *name;
+    ngx_uint_t     op;  /* ngx_thread_file_op_e */
 
     u_char        *buf;
     size_t         size;
     ngx_chain_t   *chain;
     off_t          offset;
 
+    ngx_int_t      mode;
+    ngx_int_t      create;
+    ngx_int_t      access;
+
     size_t         nbytes;
     ngx_err_t      err;
 } ngx_thread_file_ctx_t;
 
 
+ngx_int_t
+ngx_thread_open(ngx_file_t *file, u_char *name, ngx_int_t mode,
+    ngx_int_t create, ngx_int_t access, ngx_pool_t *pool)
+{
+    ngx_thread_task_t      *task;
+    ngx_thread_file_ctx_t  *ctx;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_CORE, file->log, 0,
+                   "thread open: \"%s\"", name);
+
+    task = file->thread_task;
+
+    if (task == NULL) {
+        task = ngx_thread_task_alloc(pool, sizeof(ngx_thread_file_ctx_t));
+        if (task == NULL) {
+            return NGX_ERROR;
+        }
+
+        file->thread_task = task;
+    }
+
+    ctx = task->ctx;
+
+    if (task->event.complete) {
+        task->event.complete = 0;
+
+        if (ctx->op != NGX_THREAD_FILE_OPEN) {
+            ngx_log_error(NGX_LOG_ALERT, file->log, 0,
+                          "invalid thread operation, open expected");
+            return NGX_ERROR;
+        }
+
+        if (ctx->err) {
+            file->err = ctx->err;
+            return NGX_ERROR;
+        }
+
+        file->fd = ctx->fd;
+
+        return NGX_OK;
+    }
+
+    task->handler = ngx_thread_open_handler;
+
+    ctx->op = NGX_THREAD_FILE_OPEN;
+
+    ctx->name = name;
+    ctx->mode = mode;
+    ctx->create = create;
+    ctx->access = access;
+
+    if (file->thread_handler(task, file) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    return NGX_AGAIN;
+}
+
+
+static void
+ngx_thread_open_handler(void *data, ngx_log_t *log)
+{
+    ngx_thread_file_ctx_t *ctx = data;
+
+    ngx_fd_t  fd;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "thread open handler");
+
+    fd = ngx_open_file(ctx->name, ctx->mode, ctx->create, ctx->access);
+
+    if (fd == NGX_INVALID_FILE) {
+        ctx->err = ngx_errno;
+
+    } else {
+        ctx->fd = fd;
+        ctx->err = 0;
+    }
+}
+
+
 ssize_t
 ngx_thread_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
     ngx_pool_t *pool)
@@ -118,9 +211,9 @@ ngx_thread_read(ngx_file_t *file, u_char
     if (task->event.complete) {
         task->event.complete = 0;
 
-        if (ctx->write) {
+        if (ctx->op != NGX_THREAD_FILE_READ) {
             ngx_log_error(NGX_LOG_ALERT, file->log, 0,
-                          "invalid thread call, read instead of write");
+                          "invalid thread operation, read expected");
             return NGX_ERROR;
         }
 
@@ -135,7 +228,7 @@ ngx_thread_read(ngx_file_t *file, u_char
 
     task->handler = ngx_thread_read_handler;
 
-    ctx->write = 0;
+    ctx->op = NGX_THREAD_FILE_READ;
 
     ctx->fd = file->fd;
     ctx->buf = buf;
@@ -501,9 +594,9 @@ ngx_thread_write_chain_to_file(ngx_file_
     if (task->event.complete) {
         task->event.complete = 0;
 
-        if (!ctx->write) {
+        if (ctx->op != NGX_THREAD_FILE_WRITE) {
             ngx_log_error(NGX_LOG_ALERT, file->log, 0,
-                          "invalid thread call, write instead of read");
+                          "invalid thread operation, write expected");
             return NGX_ERROR;
         }
 
@@ -519,7 +612,7 @@ ngx_thread_write_chain_to_file(ngx_file_
 
     task->handler = ngx_thread_write_chain_to_file_handler;
 
-    ctx->write = 1;
+    ctx->op = NGX_THREAD_FILE_WRITE;
 
     ctx->fd = file->fd;
     ctx->chain = cl;
diff --git a/src/os/unix/ngx_files.h b/src/os/unix/ngx_files.h
--- a/src/os/unix/ngx_files.h
+++ b/src/os/unix/ngx_files.h
@@ -386,6 +386,8 @@ extern ngx_uint_t  ngx_file_aio;
 #endif
 
 #if (NGX_THREADS)
+ngx_int_t ngx_thread_open(ngx_file_t *file, u_char *name,
+    ngx_int_t mode, ngx_int_t create, ngx_int_t access, ngx_pool_t *pool);
 ssize_t ngx_thread_read(ngx_file_t *file, u_char *buf, size_t size,
     off_t offset, ngx_pool_t *pool);
 ssize_t ngx_thread_write_chain_to_file(ngx_file_t *file, ngx_chain_t *cl,
-------------- next part --------------
# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1627978611 -10800
#      Tue Aug 03 11:16:51 2021 +0300
# Node ID 70f0a417176574d8f4739bf960fdd0b873ccf048
# Parent  f386594b298d1442e1bfe1add9cdf3c6cf26917b
Static: threaded open support.

diff --git a/src/http/modules/ngx_http_static_module.c b/src/http/modules/ngx_http_static_module.c
--- a/src/http/modules/ngx_http_static_module.c
+++ b/src/http/modules/ngx_http_static_module.c
@@ -11,7 +11,14 @@
 
 
 static ngx_int_t ngx_http_static_handler(ngx_http_request_t *r);
+static ngx_int_t ngx_http_static_send(ngx_http_request_t *r);
 static ngx_int_t ngx_http_static_init(ngx_conf_t *cf);
+#if (NGX_THREADS)
+static void ngx_http_static_write_event_handler(ngx_http_request_t *r);
+static ngx_int_t ngx_http_static_thread_handler(ngx_thread_task_t *task,
+    ngx_file_t *file);
+static void ngx_http_static_thread_event_handler(ngx_event_t *ev);
+#endif
 
 
 static ngx_http_module_t  ngx_http_static_module_ctx = {
@@ -48,15 +55,9 @@ ngx_module_t  ngx_http_static_module = {
 static ngx_int_t
 ngx_http_static_handler(ngx_http_request_t *r)
 {
-    u_char                    *last, *location;
-    size_t                     root, len;
-    ngx_str_t                  path;
+    size_t                     root;
+    u_char                    *last;
     ngx_int_t                  rc;
-    ngx_uint_t                 level;
-    ngx_log_t                 *log;
-    ngx_buf_t                 *b;
-    ngx_chain_t                out;
-    ngx_open_file_info_t       of;
     ngx_http_core_loc_conf_t  *clcf;
 
     if (!(r->method & (NGX_HTTP_GET|NGX_HTTP_HEAD|NGX_HTTP_POST))) {
@@ -67,42 +68,81 @@ ngx_http_static_handler(ngx_http_request
         return NGX_DECLINED;
     }
 
-    log = r->connection->log;
-
     /*
      * ngx_http_map_uri_to_path() allocates memory for terminating '\0'
      * so we do not need to reserve memory for '/' for possible redirect
      */
 
-    last = ngx_http_map_uri_to_path(r, &path, &root, 0);
+    last = ngx_http_map_uri_to_path(r, &r->open_file_name, &root, 0);
     if (last == NULL) {
         return NGX_HTTP_INTERNAL_SERVER_ERROR;
     }
 
-    path.len = last - path.data;
+    r->open_file_name.len = last - r->open_file_name.data;
 
-    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0,
-                   "http filename: \"%s\"", path.data);
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "http filename: \"%s\"", r->open_file_name.data);
 
     clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
 
-    ngx_memzero(&of, sizeof(ngx_open_file_info_t));
+    ngx_memzero(&r->open_file_info, sizeof(ngx_open_file_info_t));
 
-    of.read_ahead = clcf->read_ahead;
-    of.directio = clcf->directio;
-    of.valid = clcf->open_file_cache_valid;
-    of.min_uses = clcf->open_file_cache_min_uses;
-    of.errors = clcf->open_file_cache_errors;
-    of.events = clcf->open_file_cache_events;
+#if (NGX_THREADS)
+    if (clcf->aio == NGX_HTTP_AIO_THREADS && clcf->aio_open) {
+        r->open_file_info.thread_handler = ngx_http_static_thread_handler;
+        r->open_file_info.thread_ctx = r;
+    }
+#endif
 
-    if (ngx_http_set_disable_symlinks(r, clcf, &path, &of) != NGX_OK) {
+    if (ngx_http_set_open_file(r, clcf, &r->open_file_name, &r->open_file_info)
+        != NGX_OK)
+    {
         return NGX_HTTP_INTERNAL_SERVER_ERROR;
     }
 
-    if (ngx_open_cached_file(clcf->open_file_cache, &path, &of, r->pool)
-        != NGX_OK)
-    {
-        switch (of.err) {
+    rc = ngx_http_static_send(r);
+
+#if (NGX_THREADS)
+    if (rc == NGX_DONE) {
+        r->main->count++;
+        r->write_event_handler = ngx_http_static_write_event_handler;
+    }
+#endif
+
+    return rc;
+}
+
+
+static ngx_int_t
+ngx_http_static_send(ngx_http_request_t *r)
+{
+    u_char                    *location, *last;
+    size_t                     len;
+    ngx_log_t                 *log;
+    ngx_int_t                  rc;
+    ngx_uint_t                 level;
+    ngx_buf_t                 *b;
+    ngx_chain_t                out;
+    ngx_http_core_loc_conf_t  *clcf;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "http static send: \"%s\"", r->open_file_name.data);
+
+    clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
+    log = r->connection->log;
+
+    rc = ngx_open_cached_file(clcf->open_file_cache, &r->open_file_name,
+                              &r->open_file_info, r->pool);
+
+#if (NGX_THREADS)
+    if (rc == NGX_AGAIN) {
+        return NGX_DONE;
+    }
+#endif
+
+    if (rc != NGX_OK) {
+        switch (r->open_file_info.err) {
 
         case 0:
             return NGX_HTTP_INTERNAL_SERVER_ERROR;
@@ -133,8 +173,9 @@ ngx_http_static_handler(ngx_http_request
         }
 
         if (rc != NGX_HTTP_NOT_FOUND || clcf->log_not_found) {
-            ngx_log_error(level, log, of.err,
-                          "%s \"%s\" failed", of.failed, path.data);
+            ngx_log_error(level, log, r->open_file_info.err,
+                          "%s \"%s\" failed", r->open_file_info.failed,
+                          r->open_file_name.data);
         }
 
         return rc;
@@ -142,9 +183,10 @@ ngx_http_static_handler(ngx_http_request
 
     r->root_tested = !r->error_page;
 
-    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, "http static fd: %d", of.fd);
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, "http static fd: %d",
+                   r->open_file_info.fd);
 
-    if (of.is_dir) {
+    if (r->open_file_info.is_dir) {
 
         ngx_log_debug0(NGX_LOG_DEBUG_HTTP, log, 0, "http dir");
 
@@ -157,10 +199,10 @@ ngx_http_static_handler(ngx_http_request
 
         len = r->uri.len + 1;
 
-        if (!clcf->alias && r->args.len == 0) {
-            location = path.data + root;
+        if (!clcf->alias && clcf->root_lengths == NULL && r->args.len == 0) {
+            location = r->open_file_name.data + clcf->root.len;
 
-            *last = '/';
+            r->open_file_name.data[r->open_file_name.len] = '/';
 
         } else {
             if (r->args.len) {
@@ -193,9 +235,9 @@ ngx_http_static_handler(ngx_http_request
 
 #if !(NGX_WIN32) /* the not regular files are probably Unix specific */
 
-    if (!of.is_file) {
+    if (!r->open_file_info.is_file) {
         ngx_log_error(NGX_LOG_CRIT, log, 0,
-                      "\"%s\" is not a regular file", path.data);
+                      "\"%s\" is not a regular file", r->open_file_name.data);
 
         return NGX_HTTP_NOT_FOUND;
     }
@@ -215,8 +257,8 @@ ngx_http_static_handler(ngx_http_request
     log->action = "sending response to client";
 
     r->headers_out.status = NGX_HTTP_OK;
-    r->headers_out.content_length_n = of.size;
-    r->headers_out.last_modified_time = of.mtime;
+    r->headers_out.content_length_n = r->open_file_info.size;
+    r->headers_out.last_modified_time = r->open_file_info.mtime;
 
     if (ngx_http_set_etag(r) != NGX_OK) {
         return NGX_HTTP_INTERNAL_SERVER_ERROR;
@@ -226,7 +268,7 @@ ngx_http_static_handler(ngx_http_request
         return NGX_HTTP_INTERNAL_SERVER_ERROR;
     }
 
-    if (r != r->main && of.size == 0) {
+    if (r != r->main && r->open_file_info.size == 0) {
         return ngx_http_send_header(r);
     }
 
@@ -251,16 +293,16 @@ ngx_http_static_handler(ngx_http_request
     }
 
     b->file_pos = 0;
-    b->file_last = of.size;
+    b->file_last = r->open_file_info.size;
 
     b->in_file = b->file_last ? 1: 0;
     b->last_buf = (r == r->main) ? 1: 0;
     b->last_in_chain = 1;
 
-    b->file->fd = of.fd;
-    b->file->name = path;
+    b->file->fd = r->open_file_info.fd;
+    b->file->name = r->open_file_name;
     b->file->log = log;
-    b->file->directio = of.is_directio;
+    b->file->directio = r->open_file_info.is_directio;
 
     out.buf = b;
     out.next = NULL;
@@ -269,6 +311,94 @@ ngx_http_static_handler(ngx_http_request
 }
 
 
+#if (NGX_THREADS)
+
+static void
+ngx_http_static_write_event_handler(ngx_http_request_t *r)
+{
+    ngx_int_t  rc;
+
+    if (r->aio) {
+        return;
+    }
+
+    rc = ngx_http_static_send(r);
+
+    if (rc != NGX_DONE) {
+        ngx_http_finalize_request(r, rc);
+    }
+}
+
+
+static ngx_int_t
+ngx_http_static_thread_handler(ngx_thread_task_t *task, ngx_file_t *file)
+{
+    ngx_str_t                  name;
+    ngx_thread_pool_t         *tp;
+    ngx_http_request_t        *r;
+    ngx_http_core_loc_conf_t  *clcf;
+
+    r = file->thread_ctx;
+
+    clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
+    tp = clcf->thread_pool;
+
+    if (tp == NULL) {
+        if (ngx_http_complex_value(r, clcf->thread_pool_value, &name)
+            != NGX_OK)
+        {
+            return NGX_ERROR;
+        }
+
+        tp = ngx_thread_pool_get((ngx_cycle_t *) ngx_cycle, &name);
+
+        if (tp == NULL) {
+            ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                          "thread pool \"%V\" not found", &name);
+            return NGX_ERROR;
+        }
+    }
+
+    task->event.data = r;
+    task->event.handler = ngx_http_static_thread_event_handler;
+
+    if (ngx_thread_task_post(tp, task) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    r->main->blocked++;
+    r->aio = 1;
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_http_static_thread_event_handler(ngx_event_t *ev)
+{
+    ngx_connection_t    *c;
+    ngx_http_request_t  *r;
+
+    r = ev->data;
+    c = r->connection;
+
+    ngx_http_set_log_request(c->log, r);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                   "http static thread: \"%V?%V\"", &r->uri, &r->args);
+
+    r->main->blocked--;
+    r->aio = 0;
+
+    r->write_event_handler(r);
+
+    ngx_http_run_posted_requests(c);
+}
+
+#endif
+
+
 static ngx_int_t
 ngx_http_static_init(ngx_conf_t *cf)
 {
diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c
--- a/src/http/ngx_http_core_module.c
+++ b/src/http/ngx_http_core_module.c
@@ -423,6 +423,13 @@ static ngx_command_t  ngx_http_core_comm
       offsetof(ngx_http_core_loc_conf_t, aio_write),
       NULL },
 
+    { ngx_string("aio_open"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_core_loc_conf_t, aio_open),
+      NULL },
+
     { ngx_string("read_ahead"),
       NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_size_slot,
@@ -2649,6 +2656,21 @@ ngx_http_cleanup_add(ngx_http_request_t 
 
 
 ngx_int_t
+ngx_http_set_open_file(ngx_http_request_t *r,
+    ngx_http_core_loc_conf_t *clcf, ngx_str_t *path, ngx_open_file_info_t *of)
+{
+    of->read_ahead = clcf->read_ahead;
+    of->directio = clcf->directio;
+    of->valid = clcf->open_file_cache_valid;
+    of->min_uses = clcf->open_file_cache_min_uses;
+    of->errors = clcf->open_file_cache_errors;
+    of->events = clcf->open_file_cache_events;
+
+    return ngx_http_set_disable_symlinks(r, clcf, path, of);
+}
+
+
+ngx_int_t
 ngx_http_set_disable_symlinks(ngx_http_request_t *r,
     ngx_http_core_loc_conf_t *clcf, ngx_str_t *path, ngx_open_file_info_t *of)
 {
@@ -3488,6 +3510,7 @@ ngx_http_core_create_loc_conf(ngx_conf_t
     clcf->subrequest_output_buffer_size = NGX_CONF_UNSET_SIZE;
     clcf->aio = NGX_CONF_UNSET;
     clcf->aio_write = NGX_CONF_UNSET;
+    clcf->aio_open = NGX_CONF_UNSET;
 #if (NGX_THREADS)
     clcf->thread_pool = NGX_CONF_UNSET_PTR;
     clcf->thread_pool_value = NGX_CONF_UNSET_PTR;
@@ -3712,6 +3735,7 @@ ngx_http_core_merge_loc_conf(ngx_conf_t 
                               (size_t) ngx_pagesize);
     ngx_conf_merge_value(conf->aio, prev->aio, NGX_HTTP_AIO_OFF);
     ngx_conf_merge_value(conf->aio_write, prev->aio_write, 0);
+    ngx_conf_merge_value(conf->aio_open, prev->aio_open, 0);
 #if (NGX_THREADS)
     ngx_conf_merge_ptr_value(conf->thread_pool, prev->thread_pool, NULL);
     ngx_conf_merge_ptr_value(conf->thread_pool_value, prev->thread_pool_value,
diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h
--- a/src/http/ngx_http_core_module.h
+++ b/src/http/ngx_http_core_module.h
@@ -383,6 +383,7 @@ struct ngx_http_core_loc_conf_s {
     ngx_flag_t    sendfile;                /* sendfile */
     ngx_flag_t    aio;                     /* aio */
     ngx_flag_t    aio_write;               /* aio_write */
+    ngx_flag_t    aio_open;                /* aio_open */
     ngx_flag_t    tcp_nopush;              /* tcp_nopush */
     ngx_flag_t    tcp_nodelay;             /* tcp_nodelay */
     ngx_flag_t    reset_timedout_connection; /* reset_timedout_connection */
@@ -523,6 +524,8 @@ ngx_int_t ngx_http_request_body_save_fil
     ngx_chain_t *chain);
 
 
+ngx_int_t ngx_http_set_open_file(ngx_http_request_t *r,
+    ngx_http_core_loc_conf_t *clcf, ngx_str_t *path, ngx_open_file_info_t *of);
 ngx_int_t ngx_http_set_disable_symlinks(ngx_http_request_t *r,
     ngx_http_core_loc_conf_t *clcf, ngx_str_t *path, ngx_open_file_info_t *of);
 
diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h
--- a/src/http/ngx_http_request.h
+++ b/src/http/ngx_http_request.h
@@ -427,6 +427,9 @@ struct ngx_http_request_s {
 
     ngx_http_variable_value_t        *variables;
 
+    ngx_str_t                         open_file_name;
+    ngx_open_file_info_t              open_file_info;
+
 #if (NGX_PCRE)
     ngx_uint_t                        ncaptures;
     int                              *captures;
-------------- next part --------------
# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1627978716 -10800
#      Tue Aug 03 11:18:36 2021 +0300
# Node ID b506b57df4a716b253cd7e5e701fd34b67f6850c
# Parent  70f0a417176574d8f4739bf960fdd0b873ccf048
Cache: threaded open support.

diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c
--- a/src/http/ngx_http_file_cache.c
+++ b/src/http/ngx_http_file_cache.c
@@ -18,6 +18,8 @@ static void ngx_http_file_cache_lock_wai
     ngx_http_cache_t *c);
 static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r,
     ngx_http_cache_t *c);
+static ngx_int_t ngx_http_file_cache_aio_open(ngx_http_request_t *r,
+    ngx_http_cache_t *c);
 static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r,
     ngx_http_cache_t *c);
 #if (NGX_HAVE_FILE_AIO)
@@ -264,13 +266,11 @@ ngx_http_file_cache_create_key(ngx_http_
 ngx_int_t
 ngx_http_file_cache_open(ngx_http_request_t *r)
 {
-    ngx_int_t                  rc, rv;
-    ngx_uint_t                 test;
-    ngx_http_cache_t          *c;
-    ngx_pool_cleanup_t        *cln;
-    ngx_open_file_info_t       of;
-    ngx_http_file_cache_t     *cache;
-    ngx_http_core_loc_conf_t  *clcf;
+    ngx_int_t               rc;
+    ngx_uint_t              test;
+    ngx_http_cache_t       *c;
+    ngx_pool_cleanup_t     *cln;
+    ngx_http_file_cache_t  *cache;
 
     c = r->cache;
 
@@ -317,7 +317,6 @@ ngx_http_file_cache_open(ngx_http_reques
 
         c->temp_file = 1;
         test = c->exists ? 1 : 0;
-        rv = NGX_DECLINED;
 
     } else { /* rc == NGX_DECLINED */
 
@@ -329,11 +328,10 @@ ngx_http_file_cache_open(ngx_http_reques
                 return NGX_HTTP_CACHE_SCARCE;
             }
 
-            rv = NGX_HTTP_CACHE_SCARCE;
+            c->temp_file = 0;
 
         } else {
             c->temp_file = 1;
-            rv = NGX_DECLINED;
         }
     }
 
@@ -342,62 +340,10 @@ ngx_http_file_cache_open(ngx_http_reques
     }
 
     if (!test) {
-        goto done;
-    }
-
-    clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
-
-    ngx_memzero(&of, sizeof(ngx_open_file_info_t));
-
-    of.uniq = c->uniq;
-    of.valid = clcf->open_file_cache_valid;
-    of.min_uses = clcf->open_file_cache_min_uses;
-    of.events = clcf->open_file_cache_events;
-    of.directio = NGX_OPEN_FILE_DIRECTIO_OFF;
-    of.read_ahead = clcf->read_ahead;
-
-    if (ngx_open_cached_file(clcf->open_file_cache, &c->file.name, &of, r->pool)
-        != NGX_OK)
-    {
-        switch (of.err) {
-
-        case 0:
-            return NGX_ERROR;
-
-        case NGX_ENOENT:
-        case NGX_ENOTDIR:
-            goto done;
-
-        default:
-            ngx_log_error(NGX_LOG_CRIT, r->connection->log, of.err,
-                          ngx_open_file_n " \"%s\" failed", c->file.name.data);
-            return NGX_ERROR;
-        }
-    }
-
-    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                   "http file cache fd: %d", of.fd);
-
-    c->file.fd = of.fd;
-    c->file.log = r->connection->log;
-    c->uniq = of.uniq;
-    c->length = of.size;
-    c->fs_size = (of.fs_size + cache->bsize - 1) / cache->bsize;
-
-    c->buf = ngx_create_temp_buf(r->pool, c->body_start);
-    if (c->buf == NULL) {
-        return NGX_ERROR;
+        return ngx_http_file_cache_lock(r, c);
     }
 
     return ngx_http_file_cache_read(r, c);
-
-done:
-
-    if (rv == NGX_DECLINED) {
-        return ngx_http_file_cache_lock(r, c);
-    }
-
-    return rv;
 }
 
 
@@ -407,6 +353,10 @@ ngx_http_file_cache_lock(ngx_http_reques
     ngx_msec_t                 now, timer;
     ngx_http_file_cache_t     *cache;
 
+    if (!c->temp_file) {
+        return NGX_HTTP_CACHE_SCARCE;
+    }
+
     if (!c->lock) {
         return NGX_DECLINED;
     }
@@ -536,6 +486,12 @@ ngx_http_file_cache_read(ngx_http_reques
     ngx_http_file_cache_t         *cache;
     ngx_http_file_cache_header_t  *h;
 
+    rc = ngx_http_file_cache_aio_open(r, c);
+
+    if (rc != NGX_OK) {
+        return rc;
+    }
+
     n = ngx_http_file_cache_aio_read(r, c);
 
     if (n < 0) {
@@ -665,6 +621,89 @@ ngx_http_file_cache_read(ngx_http_reques
 }
 
 
+static ngx_int_t
+ngx_http_file_cache_aio_open(ngx_http_request_t *r, ngx_http_cache_t *c)
+{
+    ngx_int_t                  rc;
+    ngx_http_file_cache_t     *cache;
+    ngx_http_core_loc_conf_t  *clcf;
+
+    if (c->file.fd != NGX_INVALID_FILE) {
+        return NGX_OK;
+    }
+
+    clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
+    if (!c->reading) {
+        ngx_memzero(&r->open_file_info, sizeof(ngx_open_file_info_t));
+
+        r->open_file_info.uniq = c->uniq;
+        r->open_file_info.valid = clcf->open_file_cache_valid;
+        r->open_file_info.min_uses = clcf->open_file_cache_min_uses;
+        r->open_file_info.events = clcf->open_file_cache_events;
+        r->open_file_info.directio = NGX_OPEN_FILE_DIRECTIO_OFF;
+        r->open_file_info.read_ahead = clcf->read_ahead;
+
+#if (NGX_THREADS)
+        if (clcf->aio == NGX_HTTP_AIO_THREADS && clcf->aio_open) {
+            r->open_file_info.thread_task = c->thread_task;
+            r->open_file_info.thread_handler = ngx_http_cache_thread_handler;
+            r->open_file_info.thread_ctx = r;
+        }
+#endif
+    }
+
+    rc = ngx_open_cached_file(clcf->open_file_cache, &c->file.name,
+                              &r->open_file_info, r->pool);
+
+#if (NGX_THREADS)
+
+    if (rc == NGX_AGAIN) {
+        c->reading = 1;
+        return NGX_AGAIN;
+    }
+
+    c->reading = 0;
+
+#endif
+
+    if (rc != NGX_OK) {
+        switch (r->open_file_info.err) {
+
+        case NGX_OK:
+            return NGX_ERROR;
+
+        case NGX_ENOENT:
+        case NGX_ENOTDIR:
+            return ngx_http_file_cache_lock(r, c);
+
+        default:
+            ngx_log_error(NGX_LOG_CRIT, r->connection->log,
+                          r->open_file_info.err,
+                          ngx_open_file_n " \"%s\" failed", c->file.name.data);
+            return NGX_ERROR;
+        }
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "http file cache fd: %d", r->open_file_info.fd);
+
+    cache = c->file_cache;
+    c->file.fd = r->open_file_info.fd;
+    c->file.log = r->connection->log;
+    c->uniq = r->open_file_info.uniq;
+    c->length = r->open_file_info.size;
+    c->fs_size = (r->open_file_info.fs_size + cache->bsize - 1) / cache->bsize;
+
+    c->buf = ngx_create_temp_buf(r->pool, c->body_start);
+    if (c->buf == NULL) {
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+
 static ssize_t
 ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c)
 {
@@ -1231,6 +1270,7 @@ ngx_http_file_cache_reopen(ngx_http_requ
     ngx_shmtx_unlock(&cache->shpool->mutex);
 
     c->secondary = 1;
+    c->file.fd = NGX_INVALID_FILE;
     c->file.name.len = 0;
     c->body_start = c->buffer_size;
 


More information about the nginx-devel mailing list