[nginx] Thread pools implementation.

Valentin Bartenev vbart at nginx.com
Wed Mar 18 15:57:51 UTC 2015


details:   http://hg.nginx.org/nginx/rev/466bd63b63d1
branches:  
changeset: 6018:466bd63b63d1
user:      Valentin Bartenev <vbart at nginx.com>
date:      Sat Mar 14 17:37:07 2015 +0300
description:
Thread pools implementation.

diffstat:

 auto/configure                              |    1 +
 auto/modules                                |    6 +
 auto/options                                |    4 +
 auto/sources                                |    7 +
 auto/summary                                |    4 +
 auto/threads                                |   20 +
 src/core/ngx_core.h                         |    4 +
 src/core/ngx_thread_pool.c                  |  631 ++++++++++++++++++++++++++++
 src/core/ngx_thread_pool.h                  |   36 +
 src/event/modules/ngx_aio_module.c          |    1 +
 src/event/modules/ngx_devpoll_module.c      |    1 +
 src/event/modules/ngx_epoll_module.c        |    1 +
 src/event/modules/ngx_eventport_module.c    |    1 +
 src/event/modules/ngx_iocp_module.c         |    1 +
 src/event/modules/ngx_kqueue_module.c       |    1 +
 src/event/modules/ngx_poll_module.c         |    1 +
 src/event/modules/ngx_rtsig_module.c        |    1 +
 src/event/modules/ngx_select_module.c       |    1 +
 src/event/modules/ngx_win32_select_module.c |    1 +
 src/event/ngx_event.c                       |    2 +-
 src/event/ngx_event.h                       |    4 +
 src/os/unix/ngx_linux_config.h              |    2 +-
 src/os/unix/ngx_thread.h                    |   52 ++
 src/os/unix/ngx_thread_cond.c               |   87 +++
 src/os/unix/ngx_thread_id.c                 |   70 +++
 src/os/unix/ngx_thread_mutex.c              |  174 +++++++
 26 files changed, 1112 insertions(+), 2 deletions(-)

diffs (truncated from 1359 to 300 lines):

diff -r 83d54192e97b -r 466bd63b63d1 auto/configure
--- a/auto/configure	Fri Mar 13 19:08:27 2015 +0300
+++ b/auto/configure	Sat Mar 14 17:37:07 2015 +0300
@@ -58,6 +58,7 @@ if [ "$NGX_PLATFORM" != win32 ]; then
     . auto/unix
 fi
 
+. auto/threads
 . auto/modules
 . auto/lib/conf
 
diff -r 83d54192e97b -r 466bd63b63d1 auto/modules
--- a/auto/modules	Fri Mar 13 19:08:27 2015 +0300
+++ b/auto/modules	Sat Mar 14 17:37:07 2015 +0300
@@ -432,6 +432,12 @@ fi
 modules="$CORE_MODULES $EVENT_MODULES"
 
 
+# thread pool module should be initialized after events
+if [ $USE_THREADS = YES ]; then
+    modules="$modules $THREAD_POOL_MODULE"
+fi
+
+
 if [ $USE_OPENSSL = YES ]; then
     modules="$modules $OPENSSL_MODULE"
     CORE_DEPS="$CORE_DEPS $OPENSSL_DEPS"
diff -r 83d54192e97b -r 466bd63b63d1 auto/options
--- a/auto/options	Fri Mar 13 19:08:27 2015 +0300
+++ b/auto/options	Sat Mar 14 17:37:07 2015 +0300
@@ -190,6 +190,8 @@ do
         --without-poll_module)           EVENT_POLL=NONE            ;;
         --with-aio_module)               EVENT_AIO=YES              ;;
 
+        --with-threads)                  USE_THREADS=YES            ;;
+
         --with-file-aio)                 NGX_FILE_AIO=YES           ;;
         --with-ipv6)                     NGX_IPV6=YES               ;;
 
@@ -351,6 +353,8 @@ cat << END
   --with-poll_module                 enable poll module
   --without-poll_module              disable poll module
 
+  --with-threads                     enable thread pool support
+
   --with-file-aio                    enable file AIO support
   --with-ipv6                        enable IPv6 support
 
diff -r 83d54192e97b -r 466bd63b63d1 auto/sources
--- a/auto/sources	Fri Mar 13 19:08:27 2015 +0300
+++ b/auto/sources	Sat Mar 14 17:37:07 2015 +0300
@@ -193,6 +193,13 @@ UNIX_SRCS="$CORE_SRCS $EVENT_SRCS \
 
 POSIX_DEPS=src/os/unix/ngx_posix_config.h
 
+THREAD_POOL_MODULE=ngx_thread_pool_module
+THREAD_POOL_DEPS=src/core/ngx_thread_pool.h
+THREAD_POOL_SRCS="src/core/ngx_thread_pool.c
+                  src/os/unix/ngx_thread_cond.c
+                  src/os/unix/ngx_thread_mutex.c
+                  src/os/unix/ngx_thread_id.c"
+
 FREEBSD_DEPS="src/os/unix/ngx_freebsd_config.h src/os/unix/ngx_freebsd.h"
 FREEBSD_SRCS=src/os/unix/ngx_freebsd_init.c
 FREEBSD_SENDFILE_SRCS=src/os/unix/ngx_freebsd_sendfile_chain.c
diff -r 83d54192e97b -r 466bd63b63d1 auto/summary
--- a/auto/summary	Fri Mar 13 19:08:27 2015 +0300
+++ b/auto/summary	Sat Mar 14 17:37:07 2015 +0300
@@ -7,6 +7,10 @@ echo
 echo "Configuration summary"
 
 
+if [ $USE_THREADS = YES ]; then
+    echo "  + using threads"
+fi
+
 if [ $USE_PCRE = DISABLED ]; then
     echo "  + PCRE library is disabled"
 
diff -r 83d54192e97b -r 466bd63b63d1 auto/threads
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/auto/threads	Sat Mar 14 17:37:07 2015 +0300
@@ -0,0 +1,20 @@
+
+# Copyright (C) Nginx, Inc.
+
+
+if [ $USE_THREADS = YES ]; then
+
+    if [ "$NGX_PLATFORM" = win32 ]; then
+        cat << END
+
+$0: --with-threads is not supported on Windows
+
+END
+        exit 1
+    fi
+
+    have=NGX_THREADS . auto/have
+    CORE_DEPS="$CORE_DEPS $THREAD_POOL_DEPS"
+    CORE_SRCS="$CORE_SRCS $THREAD_POOL_SRCS"
+    CORE_LIBS="$CORE_LIBS -lpthread"
+fi
diff -r 83d54192e97b -r 466bd63b63d1 src/core/ngx_core.h
--- a/src/core/ngx_core.h	Fri Mar 13 19:08:27 2015 +0300
+++ b/src/core/ngx_core.h	Sat Mar 14 17:37:07 2015 +0300
@@ -22,6 +22,10 @@ typedef struct ngx_event_s       ngx_eve
 typedef struct ngx_event_aio_s   ngx_event_aio_t;
 typedef struct ngx_connection_s  ngx_connection_t;
 
+#if (NGX_THREADS)
+typedef struct ngx_thread_task_s  ngx_thread_task_t;
+#endif
+
 typedef void (*ngx_event_handler_pt)(ngx_event_t *ev);
 typedef void (*ngx_connection_handler_pt)(ngx_connection_t *c);
 
diff -r 83d54192e97b -r 466bd63b63d1 src/core/ngx_thread_pool.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/core/ngx_thread_pool.c	Sat Mar 14 17:37:07 2015 +0300
@@ -0,0 +1,631 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ * Copyright (C) Valentin V. Bartenev
+ * Copyright (C) Ruslan Ermilov
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_thread_pool.h>
+
+
+typedef struct {
+    ngx_array_t               pools;
+} ngx_thread_pool_conf_t;
+
+
+typedef struct {
+    ngx_thread_mutex_t        mtx;
+    ngx_uint_t                count;
+    ngx_thread_task_t        *first;
+    ngx_thread_task_t       **last;
+} ngx_thread_pool_queue_t;
+
+
+struct ngx_thread_pool_s {
+    ngx_thread_cond_t         cond;
+
+    ngx_thread_pool_queue_t   queue;
+
+    ngx_log_t                *log;
+    ngx_pool_t               *pool;
+
+    ngx_str_t                 name;
+    ngx_uint_t                threads;
+    ngx_uint_t                max_queue;
+
+    u_char                   *file;
+    ngx_uint_t                line;
+};
+
+
+static ngx_int_t ngx_thread_pool_init(ngx_thread_pool_t *tp, ngx_log_t *log,
+    ngx_pool_t *pool);
+static ngx_int_t ngx_thread_pool_queue_init(ngx_thread_pool_queue_t *queue,
+    ngx_log_t *log);
+static ngx_int_t ngx_thread_pool_queue_destroy(ngx_thread_pool_queue_t *queue,
+    ngx_log_t *log);
+static void ngx_thread_pool_destroy(ngx_thread_pool_t *tp);
+
+static void *ngx_thread_pool_cycle(void *data);
+static void ngx_thread_pool_handler(ngx_event_t *ev);
+
+static char *ngx_thread_pool(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
+
+static void *ngx_thread_pool_create_conf(ngx_cycle_t *cycle);
+static char *ngx_thread_pool_init_conf(ngx_cycle_t *cycle, void *conf);
+
+static ngx_int_t ngx_thread_pool_init_worker(ngx_cycle_t *cycle);
+static void ngx_thread_pool_exit_worker(ngx_cycle_t *cycle);
+
+
+static ngx_command_t  ngx_thread_pool_commands[] = {
+
+    { ngx_string("thread_pool"),
+      NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE23,
+      ngx_thread_pool,
+      0,
+      0,
+      NULL },
+
+      ngx_null_command
+};
+
+
+static ngx_core_module_t  ngx_thread_pool_module_ctx = {
+    ngx_string("thread_pool"),
+    ngx_thread_pool_create_conf,
+    ngx_thread_pool_init_conf
+};
+
+
+ngx_module_t  ngx_thread_pool_module = {
+    NGX_MODULE_V1,
+    &ngx_thread_pool_module_ctx,           /* module context */
+    ngx_thread_pool_commands,              /* module directives */
+    NGX_CORE_MODULE,                       /* module type */
+    NULL,                                  /* init master */
+    NULL,                                  /* init module */
+    ngx_thread_pool_init_worker,           /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    ngx_thread_pool_exit_worker,           /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+static ngx_str_t  ngx_thread_pool_default = ngx_string("default");
+
+static ngx_uint_t               ngx_thread_pool_task_id;
+static ngx_thread_pool_queue_t  ngx_thread_pool_done;
+
+
+static ngx_int_t
+ngx_thread_pool_init(ngx_thread_pool_t *tp, ngx_log_t *log, ngx_pool_t *pool)
+{
+    int             err;
+    pthread_t       tid;
+    ngx_uint_t      n;
+    pthread_attr_t  attr;
+
+    if (ngx_notify == NULL) {
+        ngx_log_error(NGX_LOG_ALERT, log, 0,
+               "the configured event method cannot be used with thread pools");
+        return NGX_ERROR;
+    }
+
+    if (ngx_thread_pool_queue_init(&tp->queue, log) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    if (ngx_thread_cond_create(&tp->cond, log) != NGX_OK) {
+        (void) ngx_thread_pool_queue_destroy(&tp->queue, log);
+        return NGX_ERROR;
+    }
+
+    tp->log = log;
+    tp->pool = pool;
+
+    err = pthread_attr_init(&attr);
+    if (err) {
+        ngx_log_error(NGX_LOG_ALERT, log, err,
+                      "pthread_attr_init() failed");
+        return NGX_ERROR;
+    }
+
+#if 0
+    err = pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
+    if (err) {
+        ngx_log_error(NGX_LOG_ALERT, log, err,
+                      "pthread_attr_setstacksize() failed");
+        return NGX_ERROR;
+    }
+#endif
+
+    for (n = 0; n < tp->threads; n++) {
+        err = pthread_create(&tid, &attr, ngx_thread_pool_cycle, tp);
+        if (err) {
+            ngx_log_error(NGX_LOG_ALERT, log, err,
+                          "pthread_create() failed");
+            return NGX_ERROR;
+        }
+    }
+
+    (void) pthread_attr_destroy(&attr);
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_thread_pool_queue_init(ngx_thread_pool_queue_t *queue, ngx_log_t *log)
+{
+    queue->count = 0;
+    queue->first = NULL;
+    queue->last = &queue->first;
+
+    return ngx_thread_mutex_create(&queue->mtx, log);
+}
+
+
+static ngx_int_t
+ngx_thread_pool_queue_destroy(ngx_thread_pool_queue_t *queue, ngx_log_t *log)
+{
+    return ngx_thread_mutex_destroy(&queue->mtx, log);
+}
+



More information about the nginx-devel mailing list