FreeBSD & Linux disk AIO support

Igor Sysoev is at rambler-co.ru
Wed Aug 26 12:15:11 MSD 2009


On Wed, Aug 26, 2009 at 11:59:05AM +0400, Dmitriy Timokhin wrote:

> 2009/8/26 Igor Sysoev <is at rambler-co.ru>:
> > Скорее всего, glibc старая. Насколько я понимаю, это дистрибутив, где
> > руками поставили ядро поновее ?
> >
> Не обязательно -- например, в debian/lenny ядро уже с eventfd (2.6.26),
> а glibc ещё без (2.7).

Новый патч.


-- 
Игорь Сысоев
http://sysoev.ru
-------------- next part --------------
Index: src/event/modules/ngx_kqueue_module.c
===================================================================
--- src/event/modules/ngx_kqueue_module.c	(revision 2362)
+++ src/event/modules/ngx_kqueue_module.c	(working copy)
@@ -7,7 +7,6 @@
 #include <ngx_config.h>
 #include <ngx_core.h>
 #include <ngx_event.h>
-#include <ngx_kqueue_module.h>
 
 
 typedef struct {
@@ -113,7 +112,6 @@
 };
 
 
-
 static ngx_int_t
 ngx_kqueue_init(ngx_cycle_t *cycle, ngx_msec_t timer)
 {
@@ -537,11 +535,7 @@
 
     events = kevent(ngx_kqueue, change_list, n, event_list, (int) nevents, tp);
 
-    if (events == -1) {
-        err = ngx_errno;
-    } else {
-        err = 0;
-    }
+    err = (events == -1) ? ngx_errno : 0;
 
     if (flags & NGX_UPDATE_TIME) {
         ngx_time_update(0, 0);
Index: src/event/modules/ngx_select_module.c
===================================================================
--- src/event/modules/ngx_select_module.c	(revision 2362)
+++ src/event/modules/ngx_select_module.c	(working copy)
@@ -260,11 +260,7 @@
 
     ready = select(max_fd + 1, &work_read_fd_set, &work_write_fd_set, NULL, tp);
 
-    if (ready == -1) {
-        err = ngx_socket_errno;
-    } else {
-        err = 0;
-    }
+    err = (ready == -1) ? ngx_errno : 0;
 
     if (flags & NGX_UPDATE_TIME) {
         ngx_time_update(0, 0);
Index: src/event/modules/ngx_win32_select_module.c
===================================================================
--- src/event/modules/ngx_win32_select_module.c	(revision 2362)
+++ src/event/modules/ngx_win32_select_module.c	(working copy)
@@ -266,11 +266,7 @@
         ready = 0;
     }
 
-    if (ready == -1) {
-        err = ngx_socket_errno;
-    } else {
-        err = 0;
-    }
+    err = (ready == -1) ? ngx_socket_errno : 0;
 
     if (flags & NGX_UPDATE_TIME) {
         ngx_time_update(0, 0);
Index: src/event/modules/ngx_kqueue_module.h
===================================================================
--- src/event/modules/ngx_kqueue_module.h	(revision 2362)
+++ src/event/modules/ngx_kqueue_module.h	(working copy)
@@ -1,16 +0,0 @@
-
-/*
- * Copyright (C) Igor Sysoev
- */
-
-
-#ifndef _NGX_KQUEUE_MODULE_H_INCLUDED_
-#define _NGX_KQUEUE_MODULE_H_INCLUDED_
-
-
-extern int                 ngx_kqueue;
-extern ngx_module_t        ngx_kqueue_module;
-extern ngx_event_module_t  ngx_kqueue_module_ctx;
-
-
-#endif /* _NGX_KQUEUE_MODULE_H_INCLUDED_ */
Index: src/event/modules/ngx_devpoll_module.c
===================================================================
--- src/event/modules/ngx_devpoll_module.c	(revision 2362)
+++ src/event/modules/ngx_devpoll_module.c	(working copy)
@@ -369,11 +369,7 @@
     dvp.dp_timeout = timer;
     events = ioctl(dp, DP_POLL, &dvp);
 
-    if (events == -1) {
-        err = ngx_errno;
-    } else {
-        err = 0;
-    }
+    err = (events == -1) ? ngx_errno : 0;
 
     if (flags & NGX_UPDATE_TIME) {
         ngx_time_update(0, 0);
Index: src/event/modules/ngx_poll_module.c
===================================================================
--- src/event/modules/ngx_poll_module.c	(revision 2362)
+++ src/event/modules/ngx_poll_module.c	(working copy)
@@ -260,11 +260,7 @@
 
     ready = poll(event_list, (u_int) nevents, (int) timer);
 
-    if (ready == -1) {
-        err = ngx_errno;
-    } else {
-        err = 0;
-    }
+    err = (ready == -1) ? ngx_errno : 0;
 
     if (flags & NGX_UPDATE_TIME) {
         ngx_time_update(0, 0);
Index: src/event/modules/ngx_aio_module.c
===================================================================
--- src/event/modules/ngx_aio_module.c	(revision 2362)
+++ src/event/modules/ngx_aio_module.c	(working copy)
@@ -7,13 +7,11 @@
 #include <ngx_config.h>
 #include <ngx_core.h>
 #include <ngx_event.h>
-#include <ngx_aio.h>
 
-#if (NGX_HAVE_KQUEUE)
-#include <ngx_kqueue_module.h>
-#endif
 
+extern ngx_event_module_t  ngx_kqueue_module_ctx;
 
+
 static ngx_int_t ngx_aio_init(ngx_cycle_t *cycle, ngx_msec_t timer);
 static void ngx_aio_done(ngx_cycle_t *cycle);
 static ngx_int_t ngx_aio_add_event(ngx_event_t *ev, ngx_int_t event,
@@ -73,7 +71,6 @@
 };
 
 
-
 #if (NGX_HAVE_KQUEUE)
 
 static ngx_int_t
Index: src/event/modules/ngx_epoll_module.c
===================================================================
--- src/event/modules/ngx_epoll_module.c	(revision 2362)
+++ src/event/modules/ngx_epoll_module.c	(working copy)
@@ -43,10 +43,6 @@
     epoll_data_t  data;
 };
 
-int epoll_create(int size);
-int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
-int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout);
-
 int epoll_create(int size)
 {
     return -1;
@@ -62,7 +58,28 @@
     return -1;
 }
 
+#if (NGX_HAVE_FILE_AIO)
+
+#define __NR_io_setup      245
+#define __NR_io_getevents  247
+
+typedef u_int  aio_context_t;
+
+struct io_event {
+    uint64_t  data;  /* the data field from the iocb */
+    uint64_t  obj;   /* what iocb this event came from */
+    int64_t   res;   /* result code for this event */
+    int64_t   res2;  /* secondary result */
+};
+
+
+int eventfd(u_int initval)
+{
+    return -1;
+}
+
 #endif
+#endif
 
 
 typedef struct {
@@ -82,6 +99,10 @@
 static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
     ngx_uint_t flags);
 
+#if (NGX_HAVE_FILE_AIO)
+static void ngx_epoll_eventfd_handler(ngx_event_t *ev);
+#endif
+
 static void *ngx_epoll_create_conf(ngx_cycle_t *cycle);
 static char *ngx_epoll_init_conf(ngx_cycle_t *cycle, void *conf);
 
@@ -89,7 +110,16 @@
 static struct epoll_event  *event_list;
 static ngx_uint_t           nevents;
 
+#if (NGX_HAVE_FILE_AIO)
 
+int                         ngx_eventfd = -1;
+aio_context_t               ngx_aio_ctx = 0;
+
+static ngx_event_t          ngx_eventfd_event;
+static ngx_connection_t     ngx_eventfd_conn;
+
+#endif
+
 static ngx_str_t      epoll_name = ngx_string("epoll");
 
 static ngx_command_t  ngx_epoll_commands[] = {
@@ -140,6 +170,25 @@
 };
 
 
+#if (NGX_HAVE_FILE_AIO)
+
+static long
+io_setup(u_int nr_reqs, aio_context_t *ctx)
+{
+    return syscall(__NR_io_setup, nr_reqs, ctx);
+}
+
+
+static long
+io_getevents(aio_context_t ctx, long min_nr, long nr, struct io_event *events,
+    struct timespec *tmo)
+{
+    return syscall(__NR_io_getevents, ctx, min_nr, nr, events, tmo);
+}
+
+#endif
+
+
 static ngx_int_t
 ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer)
 {
@@ -155,6 +204,76 @@
                           "epoll_create() failed");
             return NGX_ERROR;
         }
+
+#if (NGX_HAVE_FILE_AIO)
+        {
+        int                 n;
+        struct epoll_event  ee;
+
+#if (NGX_HAVE_EVENTFD2)
+
+        /* EFD_NONBLOCK may be not defiend */
+        ngx_eventfd = eventfd(0, O_NONBLOCK);
+
+        if (ngx_eventfd == -1) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                          "eventfd(O_NONBLOCK) failed");
+            return NGX_ERROR;
+        }
+
+#else
+
+#if (NGX_HAVE_EVENTFD)
+
+        ngx_eventfd = eventfd(0);
+
+#else
+        ngx_eventfd = syscall(SYS_eventfd, 0)
+
+#endif
+
+        if (ngx_eventfd == -1) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                          "eventfd() failed");
+            return NGX_ERROR;
+        }
+
+        n = 1;
+
+        if (ioctl(ngx_eventfd, FIONBIO, &n) == -1) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                          "ioctl(eventfd, FIONBIO) failed");
+        }
+#endif
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                       "eventfd: %d", ngx_eventfd);
+
+        n = io_setup(1024, &ngx_aio_ctx);
+
+        if (n != 0) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, -n, "io_setup() failed");
+            return NGX_ERROR;
+        }
+
+        ngx_eventfd_event.data = &ngx_eventfd_conn;
+        ngx_eventfd_event.handler = ngx_epoll_eventfd_handler;
+        ngx_eventfd_event.log = cycle->log;
+        ngx_eventfd_event.active = 1;
+        ngx_eventfd_conn.fd = ngx_eventfd;
+        ngx_eventfd_conn.read = &ngx_eventfd_event;
+        ngx_eventfd_conn.log = cycle->log;
+
+        ee.events = EPOLLIN|EPOLLET;
+        ee.data.ptr = &ngx_eventfd_conn;
+
+        if (epoll_ctl(ep, EPOLL_CTL_ADD, ngx_eventfd, &ee) == -1) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                          "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed");
+            return NGX_ERROR;
+        }
+        }
+#endif
     }
 
     if (nevents < epcf->events) {
@@ -401,11 +520,7 @@
 
     events = epoll_wait(ep, event_list, (int) nevents, timer);
 
-    if (events == -1) {
-        err = ngx_errno;
-    } else {
-        err = 0;
-    }
+    err = (events == -1) ? ngx_errno : 0;
 
     if (flags & NGX_UPDATE_TIME) {
         ngx_time_update(0, 0);
@@ -545,6 +660,90 @@
 }
 
 
+#if (NGX_HAVE_FILE_AIO)
+
+static void
+ngx_epoll_eventfd_handler(ngx_event_t *ev)
+{
+    int               n;
+    long              i, events;
+    uint64_t          ready;
+    ngx_err_t         err;
+    ngx_event_t      *e;
+    struct io_event   event[64];
+    struct timespec   ts;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd handler");
+
+    n = read(ngx_eventfd, &ready, 8);
+
+    err = ngx_errno;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd: %d", n);
+
+    if (n != 8) {
+        if (n == -1) {
+            if (err == NGX_EAGAIN) {
+                return;
+            }
+
+            ngx_log_error(NGX_LOG_ALERT, ev->log, err, "read(eventfd) failed");
+            return;
+        }
+
+        ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                      "read(eventfd) returned only %d bytes", n);
+        return;
+    }
+
+    ts.tv_sec = 0;
+    ts.tv_nsec = 0;
+
+    while (ready) {
+
+        events = io_getevents(ngx_aio_ctx, 1, 64, event, &ts);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                       "io_getevents: %l", events);
+
+        if (events > 0) {
+            ready -= events;
+
+            for (i = 0; i < events; i++) {
+
+                ngx_log_debug4(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                               "io_event: %uXL %uXL %L %L",
+                                event[i].data, event[i].obj,
+                                event[i].res, event[i].res2);
+
+                e = (ngx_event_t *) (uintptr_t) event[i].data;
+
+                e->complete = 1;
+                e->ready = 1;
+                e->aio_res = event[i].res;
+
+                ngx_post_event(e, &ngx_posted_events);
+            }
+
+            continue;
+        }
+
+        if (events == 0) {
+            ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                          "io_getevents() did not return %uL events", ready);
+            return;
+        }
+
+        if (events < 0) {
+            ngx_log_error(NGX_LOG_ALERT, ev->log, -events,
+                          "io_getevents() failed");
+        }
+    }
+}
+
+#endif
+
+
 static void *
 ngx_epoll_create_conf(ngx_cycle_t *cycle)
 {
Index: src/event/ngx_event.h
===================================================================
--- src/event/ngx_event.h	(revision 2365)
+++ src/event/ngx_event.h	(working copy)
@@ -67,6 +67,7 @@
     unsigned         timer_set:1;
 
     unsigned         delayed:1;
+    unsigned         blocked:1;
 
     unsigned         read_discarded:1;
 
@@ -93,6 +94,10 @@
     int              kq_errno;
 #endif
 
+#if (NGX_HAVE_EPOLL && NGX_HAVE_FILE_AIO)
+    int              aio_res;
+#endif
+
     /*
      * kqueue only:
      *   accept:     number of sockets that wait to be accepted
Index: src/os/unix/ngx_aio_write.c
===================================================================
--- src/os/unix/ngx_aio_write.c	(revision 2362)
+++ src/os/unix/ngx_aio_write.c	(working copy)
@@ -7,20 +7,10 @@
 #include <ngx_config.h>
 #include <ngx_core.h>
 #include <ngx_event.h>
-#include <ngx_aio.h>
 
-#if (NGX_HAVE_KQUEUE)
-#include <ngx_kqueue_module.h>
-#endif
 
+extern int  ngx_kqueue;
 
-/*
- * the ready data requires 3 syscalls:
- *     aio_write(), aio_error(), aio_return()
- * the non-ready data requires 4 (kqueue) or 5 syscalls:
- *     aio_write(), aio_error(), notifiction, aio_error(), aio_return()
- *                               timeout, aio_cancel(), aio_error()
- */
 
 ssize_t
 ngx_aio_write(ngx_connection_t *c, u_char *buf, size_t size)
Index: src/os/unix/ngx_process.c
===================================================================
--- src/os/unix/ngx_process.c	(revision 2362)
+++ src/os/unix/ngx_process.c	(working copy)
@@ -73,6 +73,8 @@
 
     { SIGCHLD, "SIGCHLD", "", ngx_signal_handler },
 
+    { SIGSYS, "SIGSYS, SIG_IGN", "", SIG_IGN },
+
     { SIGPIPE, "SIGPIPE, SIG_IGN", "", SIG_IGN },
 
     { 0, NULL, "", NULL }
Index: src/os/unix/ngx_os.h
===================================================================
--- src/os/unix/ngx_os.h	(revision 2362)
+++ src/os/unix/ngx_os.h	(working copy)
@@ -47,7 +47,15 @@
 ngx_chain_t *ngx_writev_chain(ngx_connection_t *c, ngx_chain_t *in,
     off_t limit);
 
+#if (NGX_HAVE_AIO)
+ssize_t ngx_aio_read(ngx_connection_t *c, u_char *buf, size_t size);
+ssize_t ngx_aio_read_chain(ngx_connection_t *c, ngx_chain_t *cl);
+ssize_t ngx_aio_write(ngx_connection_t *c, u_char *buf, size_t size);
+ngx_chain_t *ngx_aio_write_chain(ngx_connection_t *c, ngx_chain_t *in,
+    off_t limit);
+#endif
 
+
 extern ngx_os_io_t  ngx_os_io;
 extern ngx_int_t    ngx_ncpu;
 extern ngx_int_t    ngx_max_sockets;
Index: src/os/unix/ngx_aio_read.c
===================================================================
--- src/os/unix/ngx_aio_read.c	(revision 2362)
+++ src/os/unix/ngx_aio_read.c	(working copy)
@@ -7,20 +7,10 @@
 #include <ngx_config.h>
 #include <ngx_core.h>
 #include <ngx_event.h>
-#include <ngx_aio.h>
 
-#if (NGX_HAVE_KQUEUE)
-#include <ngx_kqueue_module.h>
-#endif
 
+extern int  ngx_kqueue;
 
-/*
- * the ready data requires 3 syscalls:
- *     aio_write(), aio_error(), aio_return()
- * the non-ready data requires 4 (kqueue) or 5 syscalls:
- *     aio_write(), aio_error(), notifiction, aio_error(), aio_return()
- *                               timeout, aio_cancel(), aio_error()
- */
 
 ssize_t
 ngx_aio_read(ngx_connection_t *c, u_char *buf, size_t size)
Index: src/os/unix/ngx_aio.h
===================================================================
--- src/os/unix/ngx_aio.h	(revision 2362)
+++ src/os/unix/ngx_aio.h	(working copy)
@@ -1,21 +0,0 @@
-
-/*
- * Copyright (C) Igor Sysoev
- */
-
-
-#ifndef _NGX_AIO_H_INCLUDED_
-#define _NGX_AIO_H_INCLUDED_
-
-
-#include <ngx_core.h>
-
-
-ssize_t ngx_aio_read(ngx_connection_t *c, u_char *buf, size_t size);
-ssize_t ngx_aio_read_chain(ngx_connection_t *c, ngx_chain_t *cl);
-ssize_t ngx_aio_write(ngx_connection_t *c, u_char *buf, size_t size);
-ngx_chain_t *ngx_aio_write_chain(ngx_connection_t *c, ngx_chain_t *in,
-                                 off_t limit);
-
-
-#endif /* _NGX_AIO_H_INCLUDED_ */
Index: src/os/unix/ngx_aio_write_chain.c
===================================================================
--- src/os/unix/ngx_aio_write_chain.c	(revision 2362)
+++ src/os/unix/ngx_aio_write_chain.c	(working copy)
@@ -7,7 +7,6 @@
 #include <ngx_config.h>
 #include <ngx_core.h>
 #include <ngx_event.h>
-#include <ngx_aio.h>
 
 
 ngx_chain_t *
Index: src/os/unix/ngx_aio_read_chain.c
===================================================================
--- src/os/unix/ngx_aio_read_chain.c	(revision 2362)
+++ src/os/unix/ngx_aio_read_chain.c	(working copy)
@@ -7,7 +7,6 @@
 #include <ngx_config.h>
 #include <ngx_core.h>
 #include <ngx_event.h>
-#include <ngx_aio.h>
 
 
 ssize_t
Index: src/os/unix/ngx_file_aio_read.c
===================================================================
--- src/os/unix/ngx_file_aio_read.c	(revision 0)
+++ src/os/unix/ngx_file_aio_read.c	(revision 0)
@@ -0,0 +1,127 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+extern int  ngx_kqueue;
+
+
+ssize_t
+ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset)
+{
+    int                    n;
+    ngx_event_t           *ev;
+    ngx_file_aio_ident_t  *aio;
+    static ngx_uint_t      enosys = 0;
+
+    if (enosys) {
+        return NGX_BUSY;
+    }
+
+    aio = file->aio;
+    ev = aio->event;
+
+    if (!ev->ready) {
+        ngx_log_error(NGX_LOG_ALERT, file->log, 0, "second aio post");
+        return NGX_AGAIN;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0,
+                   "complete:%d size:%z", ev->complete, size);
+
+    if (!ev->complete) {
+        ngx_memzero(&aio->aiocb, sizeof(struct aiocb));
+
+        aio->aiocb.aio_fildes = file->fd;
+        aio->aiocb.aio_offset = offset;
+        aio->aiocb.aio_buf = buf;
+        aio->aiocb.aio_nbytes = size;
+
+#if (NGX_HAVE_KQUEUE)
+        aio->aiocb.aio_sigevent.sigev_notify_kqueue = ngx_kqueue;
+        aio->aiocb.aio_sigevent.sigev_notify = SIGEV_KEVENT;
+        aio->aiocb.aio_sigevent.sigev_value.sigval_ptr = ev;
+#endif
+
+        n = aio_read(&aio->aiocb);
+
+        if (n == -1) {
+            n = ngx_errno;
+
+            if (n == NGX_EAGAIN) {
+                return NGX_BUSY;
+            }
+
+            if (n == NGX_ENOSYS) {
+                enosys = 1;
+                return NGX_BUSY;
+            }
+
+            ngx_log_error(NGX_LOG_CRIT, file->log, n, "aio_read() failed");
+            return NGX_ERROR;
+        }
+
+        ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0,
+                       "aio_read: fd:%d %d", file->fd, n);
+
+        ev->active = 1;
+        ev->ready = 0;
+    }
+
+    ev->complete = 0;
+
+    n = aio_error(&aio->aiocb);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0,
+                   "aio_error: fd:%d %d", file->fd, n);
+
+    if (n == -1) {
+        ngx_log_error(NGX_LOG_ALERT, file->log, ngx_errno,
+                      "aio_error() failed");
+        return NGX_ERROR;
+    }
+
+    if (n != 0) {
+        if (n == NGX_EINPROGRESS) {
+            if (ev->ready) {
+                ngx_log_error(NGX_LOG_ALERT, file->log, n,
+                              "aio_read() still in progress");
+                ev->ready = 0;
+            }
+            return NGX_AGAIN;
+        }
+
+        ngx_log_error(NGX_LOG_CRIT, file->log, n, "aio_read() failed");
+        ev->ready = 0;
+        return NGX_ERROR;
+    }
+
+    n = aio_return(&aio->aiocb);
+
+    if (n == -1) {
+        ngx_log_error(NGX_LOG_ALERT, file->log, ngx_errno,
+                      "aio_return() failed");
+        ev->ready = 0;
+        return NGX_ERROR;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0,
+                   "aio_return: fd:%d %d", file->fd, n);
+
+    if (n == 0) {
+        ev->ready = 0;
+
+    } else {
+        ev->ready = 1;
+    }
+
+    ev->active = 0;
+
+    return n;
+}
Index: src/os/unix/ngx_linux_aio_read.c
===================================================================
--- src/os/unix/ngx_linux_aio_read.c	(revision 0)
+++ src/os/unix/ngx_linux_aio_read.c	(revision 0)
@@ -0,0 +1,92 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+extern int            ngx_eventfd;
+extern aio_context_t  ngx_aio_ctx;
+
+
+static long
+io_submit(aio_context_t ctx, long n, struct iocb **paiocb)
+{
+    return syscall(__NR_io_submit, ctx, n, paiocb);
+}
+
+
+ssize_t
+ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset)
+{
+    long                   n;
+    struct iocb           *piocb[1];
+    ngx_event_t           *ev;
+    ngx_file_aio_ident_t  *aio;
+    static ngx_uint_t      enosys = 0;
+
+    if (enosys) {
+        return NGX_BUSY;
+    }
+
+    aio = file->aio;
+    ev = aio->event;
+
+    if (!ev->ready) {
+        ngx_log_error(NGX_LOG_ALERT, file->log, 0, "second aio post");
+        return NGX_AGAIN;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0,
+                   "complete:%d size:%z", ev->complete, size);
+
+    if (ev->complete) {
+        ev->active = 0;
+        ev->complete = 0;
+
+        if (ev->aio_res >= 0) {
+            return ev->aio_res;
+        }
+
+        ngx_set_errno(-ev->aio_res);
+        return NGX_ERROR;
+    }
+
+    ngx_memzero(&aio->aiocb, sizeof(struct iocb));
+
+    aio->aiocb.aio_data = (uint64_t) (uintptr_t) ev;
+    aio->aiocb.aio_lio_opcode = IOCB_CMD_PREAD;
+    aio->aiocb.aio_fildes = file->fd;
+    aio->aiocb.aio_buf = (uint64_t) (uintptr_t) buf;
+    aio->aiocb.aio_nbytes = size;
+    aio->aiocb.aio_offset = offset;
+    aio->aiocb.aio_flags = IOCB_FLAG_RESFD;
+    aio->aiocb.aio_resfd = ngx_eventfd;
+
+    piocb[0] = &aio->aiocb;
+
+    n = io_submit(ngx_aio_ctx, 1, piocb);
+
+    if (n == 1) {
+        return NGX_AGAIN;
+    }
+
+    n = -n;
+
+    if (n == NGX_EAGAIN) {
+        return NGX_BUSY;
+    }
+
+    if (n == NGX_ENOSYS) {
+        enosys = 1;
+        return NGX_BUSY;
+    }
+
+    ngx_log_error(NGX_LOG_CRIT, file->log, n, "io_submit() failed");
+
+    return NGX_ERROR;
+}
Index: src/os/unix/ngx_posix_config.h
===================================================================
--- src/os/unix/ngx_posix_config.h	(revision 2365)
+++ src/os/unix/ngx_posix_config.h	(working copy)
@@ -112,6 +112,12 @@
 #endif
 
 
+#if (NGX_HAVE_FILE_AIO)
+#include <aio.h>
+typedef struct aiocb  ngx_aiocb_t;
+#endif
+
+
 #define NGX_LISTEN_BACKLOG  511
 
 
Index: src/os/unix/ngx_linux_config.h
===================================================================
--- src/os/unix/ngx_linux_config.h	(revision 2365)
+++ src/os/unix/ngx_linux_config.h	(working copy)
@@ -81,6 +81,14 @@
 #endif
 
 
+#if (NGX_HAVE_FILE_AIO)
+#include <sys/eventfd.h>
+#include <sys/syscall.h>
+#include <linux/aio_abi.h>
+typedef struct iocb  ngx_aiocb_t;
+#endif
+
+
 #define NGX_LISTEN_BACKLOG        511
 
 
Index: src/os/unix/ngx_freebsd_config.h
===================================================================
--- src/os/unix/ngx_freebsd_config.h	(revision 2365)
+++ src/os/unix/ngx_freebsd_config.h	(working copy)
@@ -73,13 +73,14 @@
 #endif
 
 
-#if (NGX_HAVE_AIO)
-#include <aio.h>
+#if (NGX_HAVE_KQUEUE)
+#include <sys/event.h>
 #endif
 
 
-#if (NGX_HAVE_KQUEUE)
-#include <sys/event.h>
+#if (NGX_HAVE_FILE_AIO || NGX_HAVE_AIO)
+#include <aio.h>
+typedef struct aiocb  ngx_aiocb_t;
 #endif
 
 
Index: src/os/unix/ngx_solaris_config.h
===================================================================
--- src/os/unix/ngx_solaris_config.h	(revision 2365)
+++ src/os/unix/ngx_solaris_config.h	(working copy)
@@ -62,16 +62,6 @@
 #endif
 
 
-#if (NGX_HAVE_SENDFILE)
-#include <sys/sendfile.h>
-#endif
-
-
-#if (NGX_HAVE_AIO)
-#include <aio.h>
-#endif
-
-
 #if (NGX_HAVE_DEVPOLL)
 #include <sys/ioctl.h>
 #include <sys/devpoll.h>
@@ -83,6 +73,11 @@
 #endif
 
 
+#if (NGX_HAVE_SENDFILE)
+#include <sys/sendfile.h>
+#endif
+
+
 #define NGX_LISTEN_BACKLOG           511
 
 
Index: src/os/unix/ngx_files.h
===================================================================
--- src/os/unix/ngx_files.h	(revision 2365)
+++ src/os/unix/ngx_files.h	(working copy)
@@ -287,4 +287,12 @@
 #define ngx_set_stderr_n         "dup2(STDERR_FILENO)"
 
 
+#if (NGX_HAVE_FILE_AIO)
+
+ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size,
+    off_t offset);
+
+#endif
+
+
 #endif /* _NGX_FILES_H_INCLUDED_ */
Index: src/http/ngx_http_copy_filter_module.c
===================================================================
--- src/http/ngx_http_copy_filter_module.c	(revision 2365)
+++ src/http/ngx_http_copy_filter_module.c	(working copy)
@@ -11,9 +11,18 @@
 
 typedef struct {
     ngx_bufs_t  bufs;
+#if (NGX_HAVE_FILE_AIO)
+    ngx_flag_t  aio;
+#endif
 } ngx_http_copy_filter_conf_t;
 
 
+#if (NGX_HAVE_FILE_AIO)
+static void ngx_http_copy_aio_handler(ngx_output_chain_ctx_t *ctx,
+    ngx_file_t *file);
+static void ngx_http_copy_aio_event_handler(ngx_event_t *ev);
+#endif
+
 static void *ngx_http_copy_filter_create_conf(ngx_conf_t *cf);
 static char *ngx_http_copy_filter_merge_conf(ngx_conf_t *cf,
     void *parent, void *child);
@@ -29,6 +38,17 @@
       offsetof(ngx_http_copy_filter_conf_t, bufs),
       NULL },
 
+#if (NGX_HAVE_FILE_AIO)
+
+    { ngx_string("aio"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_copy_filter_conf_t, aio),
+      NULL },
+
+#endif
+
       ngx_null_command
 };
 
@@ -104,6 +124,12 @@
         ctx->output_filter = (ngx_output_chain_filter_pt) ngx_http_next_filter;
         ctx->filter_ctx = r;
 
+#if (NGX_HAVE_FILE_AIO)
+        if (conf->aio) {
+            ctx->aio = ngx_http_copy_aio_handler;
+        }
+#endif
+
         r->request_output = 1;
     }
 
@@ -125,6 +151,37 @@
 }
 
 
+#if (NGX_HAVE_FILE_AIO)
+
+static void
+ngx_http_copy_aio_handler(ngx_output_chain_ctx_t *ctx, ngx_file_t *file)
+{
+    ngx_http_request_t *r;
+
+    r = ctx->filter_ctx;
+
+    file->aio->event->data = r;
+    file->aio->event->handler = ngx_http_copy_aio_event_handler;
+
+    r->connection->write->blocked = 1;
+}
+
+
+static void
+ngx_http_copy_aio_event_handler(ngx_event_t *ev)
+{
+    ngx_http_request_t  *r;
+
+    r = ev->data;
+
+    r->connection->write->blocked = 0;
+
+    r->connection->write->handler(r->connection->write);
+}
+
+#endif
+
+
 static void *
 ngx_http_copy_filter_create_conf(ngx_conf_t *cf)
 {
@@ -136,6 +193,9 @@
     }
 
     conf->bufs.num = 0;
+#if (NGX_HAVE_FILE_AIO)
+    conf->aio = NGX_CONF_UNSET;
+#endif
 
     return conf;
 }
@@ -148,6 +208,9 @@
     ngx_http_copy_filter_conf_t *conf = child;
 
     ngx_conf_merge_bufs_value(conf->bufs, prev->bufs, 1, 32768);
+#if (NGX_HAVE_FILE_AIO)
+    ngx_conf_merge_value(conf->aio, prev->aio, 0);
+#endif
 
     return NULL;
 }
Index: src/http/ngx_http_request.c
===================================================================
--- src/http/ngx_http_request.c	(revision 2365)
+++ src/http/ngx_http_request.c	(working copy)
@@ -1868,6 +1868,11 @@
             return;
         }
 
+        if (c->write->blocked) {
+            (void) ngx_http_set_write_handler(r);
+            return;
+        }
+
         ngx_http_close_request(r, 0);
         return;
     }
@@ -1966,7 +1971,7 @@
         return;
     }
 
-    if (r->buffered || c->buffered || r->postponed) {
+    if (r->buffered || c->buffered || r->postponed || c->write->blocked) {
 
         if (ngx_http_set_write_handler(r) != NGX_OK) {
             ngx_http_close_request(r, 0);
@@ -2100,7 +2105,7 @@
         }
 
     } else {
-        if (wev->delayed) {
+        if (wev->delayed || wev->blocked) {
             ngx_log_debug0(NGX_LOG_DEBUG_HTTP, wev->log, 0,
                            "http writer delayed");
 
Index: src/core/ngx_output_chain.c
===================================================================
--- src/core/ngx_output_chain.c	(revision 2365)
+++ src/core/ngx_output_chain.c	(working copy)
@@ -28,6 +28,11 @@
 #define NGX_NONE            1
 
 
+#if (NGX_HAVE_FILE_AIO)
+static ssize_t ngx_aio_read_file(ngx_output_chain_ctx_t *ctx, ngx_buf_t *src,
+    ngx_buf_t *dst, size_t size);
+#endif
+
 static ngx_inline ngx_int_t
     ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
 static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
@@ -519,8 +524,30 @@
 
 #endif
 
+#if (NGX_HAVE_FILE_AIO)
+
+        if (ctx->aio) {
+            n = ngx_aio_read_file(ctx, src, dst, size);
+
+            if (n == NGX_AGAIN) {
+                return (ngx_int_t) n;
+            }
+
+        } else {
+            n = NGX_BUSY;
+        }
+
+        if (n == NGX_BUSY) {
+            n = ngx_read_file(src->file, dst->pos, (size_t) size,
+                              src->file_pos);
+        }
+
+#else
+
         n = ngx_read_file(src->file, dst->pos, (size_t) size, src->file_pos);
 
+#endif
+
 #if (NGX_HAVE_ALIGNED_DIRECTIO)
 
         if (ctx->unaligned) {
@@ -545,12 +572,6 @@
             return (ngx_int_t) n;
         }
 
-#if (NGX_FILE_AIO_READ)
-        if (n == NGX_AGAIN) {
-            return (ngx_int_t) n;
-        }
-#endif
-
         if (n != size) {
             ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
                           ngx_read_file_n " read only %z of %O from \"%s\"",
@@ -585,6 +606,45 @@
 }
 
 
+#if (NGX_HAVE_FILE_AIO)
+
+static ssize_t
+ngx_aio_read_file(ngx_output_chain_ctx_t *ctx, ngx_buf_t *src, ngx_buf_t *dst,
+    size_t size)
+{
+    ssize_t      n;
+    ngx_file_t  *file;
+
+    file = src->file;
+
+    if (file->aio == NULL) {
+        file->aio = ngx_pcalloc(ctx->pool, sizeof(ngx_file_aio_ident_t));
+        if (file->aio == NULL) {
+            return NGX_ERROR;
+        }
+
+        file->aio->event = ngx_pcalloc(ctx->pool, sizeof(ngx_event_t));
+        if (file->aio->event == NULL) {
+            return NGX_ERROR;
+        }
+
+        file->aio->event->ready = 1;
+        file->aio->event->log = file->log;
+        file->aio->fd = file->fd;
+    }
+
+    n = ngx_file_aio_read(src->file, dst->pos, size, src->file_pos);
+
+    if (n == NGX_AGAIN) {
+        ctx->aio(ctx, src->file);
+    }
+
+    return n;
+}
+
+#endif
+
+
 ngx_int_t
 ngx_chain_writer(void *data, ngx_chain_t *in)
 {
Index: src/core/ngx_buf.h
===================================================================
--- src/core/ngx_buf.h	(revision 2365)
+++ src/core/ngx_buf.h	(working copy)
@@ -67,9 +67,16 @@
 } ngx_bufs_t;
 
 
+typedef struct ngx_output_chain_ctx_s  ngx_output_chain_ctx_t;
+
 typedef ngx_int_t (*ngx_output_chain_filter_pt)(void *ctx, ngx_chain_t *in);
 
-typedef struct {
+#if (NGX_HAVE_FILE_AIO)
+typedef void (*ngx_output_chain_aio_pt)(ngx_output_chain_ctx_t *ctx,
+    ngx_file_t *file);
+#endif
+
+struct ngx_output_chain_ctx_s {
     ngx_buf_t                   *buf;
     ngx_chain_t                 *in;
     ngx_chain_t                 *free;
@@ -90,9 +97,13 @@
 
     ngx_output_chain_filter_pt   output_filter;
     void                        *filter_ctx;
-} ngx_output_chain_ctx_t;
 
+#if (NGX_HAVE_FILE_AIO)
+    ngx_output_chain_aio_pt      aio;
+#endif
+};
 
+
 typedef struct {
     ngx_chain_t                 *out;
     ngx_chain_t                **last;
Index: src/core/ngx_file.h
===================================================================
--- src/core/ngx_file.h	(revision 2365)
+++ src/core/ngx_file.h	(working copy)
@@ -12,6 +12,22 @@
 #include <ngx_core.h>
 
 
+#if (NGX_HAVE_FILE_AIO)
+
+typedef struct {
+    ngx_connection_t          *connection;
+
+    /* STUB: event is pointer because ngx_event_s definition is incomplete */
+    ngx_event_t               *event;
+    void                      *dummy;
+    ngx_fd_t                   fd;
+
+    ngx_aiocb_t                aiocb;
+} ngx_file_aio_ident_t;
+
+#endif
+
+
 struct ngx_file_s {
     ngx_fd_t                   fd;
     ngx_str_t                  name;
@@ -22,10 +38,15 @@
 
     ngx_log_t                 *log;
 
+#if (NGX_HAVE_FILE_AIO)
+    ngx_file_aio_ident_t      *aio;
+#endif
+
     unsigned                   valid_info:1;
     unsigned                   directio:1;
 };
 
+
 #define NGX_MAX_PATH_LEVEL  3
 
 
Index: auto/os/features
===================================================================
--- auto/os/features	(revision 2365)
+++ auto/os/features	(working copy)
@@ -274,3 +274,71 @@
         CORE_LIBS="$CORE_LIBS -lrt"
     fi
 fi
+
+
+if [ $NGX_FILE_AIO = YES ]; then
+    ngx_feature="kqueue AIO support"
+    ngx_feature_name="NGX_HAVE_FILE_AIO"
+    ngx_feature_run=no
+    ngx_feature_incs="#include <aio.h>"
+    ngx_feature_path=
+    ngx_feature_libs=
+    ngx_feature_test="int  n; struct aiocb  iocb;
+                      iocb.aio_sigevent.sigev_notify = SIGEV_KEVENT;
+                      n = aio_read(&iocb)"
+    . auto/feature
+
+    if [ $ngx_found = yes ]; then
+        CORE_SRCS="$CORE_SRCS $FILE_AIO_SRCS"
+    fi
+fi
+
+
+if [ $NGX_FILE_AIO = YES ]; then
+    ngx_feature="Linux AIO support"
+    ngx_feature_name="NGX_HAVE_FILE_AIO"
+    ngx_feature_run=no
+    ngx_feature_incs="#include <linux/aio_abi.h>"
+    ngx_feature_path=
+    ngx_feature_libs=
+    ngx_feature_test="int  n; struct iocb  iocb;
+                      iocb.aio_lio_opcode = IOCB_CMD_PREAD;
+                      iocb.aio_flags = IOCB_FLAG_RESFD;
+                      iocb.aio_resfd = -1;
+                      n = eventfd(0)"
+    . auto/feature
+
+    if [ $ngx_found = yes ]; then
+        CORE_SRCS="$CORE_SRCS $LINUX_AIO_SRCS"
+    fi
+fi
+
+
+ngx_feature="eventfd2()"
+ngx_feature_name="NGX_HAVE_EVENTFD2"
+ngx_feature_run=no
+ngx_feature_incs="#include <sys/eventfd.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="int  n = eventfd(0, O_NONBLOCK)"
+. auto/feature
+
+
+ngx_feature="eventfd()"
+ngx_feature_name="NGX_HAVE_EVENTFD"
+ngx_feature_run=no
+ngx_feature_incs="#include <sys/eventfd.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="int  n = eventfd(0)"
+. auto/feature
+
+
+ngx_feature="eventfd() syscall"
+ngx_feature_name="NGX_HAVE_SYS_EVENTFD"
+ngx_feature_run=no
+ngx_feature_incs="#include <sys/syscall.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="int  n = syscall(SYS_eventfd, 0)"
+. auto/feature
Index: auto/sources
===================================================================
--- auto/sources	(revision 2365)
+++ auto/sources	(working copy)
@@ -125,6 +125,8 @@
           src/os/unix/ngx_aio_read_chain.c \
           src/os/unix/ngx_aio_write_chain.c"
 
+FILE_AIO_SRCS="src/os/unix/ngx_file_aio_read.c"
+LINUX_AIO_SRCS="src/os/unix/ngx_linux_aio_read.c"
 
 UNIX_INCS="$CORE_INCS $EVENT_INCS src/os/unix"
 
Index: auto/options
===================================================================
--- auto/options	(revision 2365)
+++ auto/options	(working copy)
@@ -43,6 +43,7 @@
 
 USE_THREADS=NO
 
+NGX_FILE_AIO=NO
 NGX_IPV6=NO
 
 HTTP=YES
@@ -170,6 +171,7 @@
         #--with-threads=*)                USE_THREADS="$value"       ;;
         #--with-threads)                  USE_THREADS="pthreads"     ;;
 
+        --with-file-aio)                 NGX_FILE_AIO=YES           ;;
         --with-ipv6)                     NGX_IPV6=YES               ;;
 
         --without-http)                  HTTP=NO                    ;;
@@ -305,6 +307,7 @@
   --with-poll_module                 enable poll module
   --without-poll_module              disable poll module
 
+  --with-file-aio                    enable file aio support
   --with-ipv6                        enable ipv6 support
 
   --with-http_ssl_module             enable ngx_http_ssl_module


More information about the nginx-ru mailing list