[PATCH] The auto parameter of the worker_processes supports to detect the container environment.

agile6v agile6v at agile6v.com
Tue Apr 24 16:47:13 UTC 2018


# HG changeset patch
# User Agile6v <agile6v at agile6v.com>
# Date 1524585905 -28800
#      Wed Apr 25 00:05:05 2018 +0800
# Node ID 89793df28d1bcf2baf00e389e6806d32d7435886
# Parent  7c614ef3c6ea330c62630d5065f961a27d0f82cd
The auto parameter of the worker_processes supports to detect the container environment.


Docker mounts cgroup information into container starting with version 1.8,
so it is possible to determine the appropriate number of CPUs based on the 
cgroup information in the container. Refer to JDK related implementation:
https://bugs.openjdk.java.net/browse/JDK-8146115


diff -r 7c614ef3c6ea -r 89793df28d1b auto/sources
--- a/auto/sources	Wed Apr 18 16:11:41 2018 +0300
+++ b/auto/sources	Wed Apr 25 00:05:05 2018 +0800
@@ -192,8 +192,8 @@
 FREEBSD_SRCS=src/os/unix/ngx_freebsd_init.c
 FREEBSD_SENDFILE_SRCS=src/os/unix/ngx_freebsd_sendfile_chain.c
 
-LINUX_DEPS="src/os/unix/ngx_linux_config.h src/os/unix/ngx_linux.h"
-LINUX_SRCS=src/os/unix/ngx_linux_init.c
+LINUX_DEPS="src/os/unix/ngx_linux_config.h src/os/unix/ngx_linux.h src/os/unix/ngx_container.h"
+LINUX_SRCS="src/os/unix/ngx_linux_init.c src/os/unix/ngx_container.c"
 LINUX_SENDFILE_SRCS=src/os/unix/ngx_linux_sendfile_chain.c
 
 
diff -r 7c614ef3c6ea -r 89793df28d1b src/os/unix/ngx_container.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/os/unix/ngx_container.c	Wed Apr 25 00:05:05 2018 +0800
@@ -0,0 +1,418 @@
+
+/*
+ * Copyright (C) agile6v
+ * Copyright (C) Xiaomi, Inc.
+ */
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+#define NGX_BUFFER_SIZE     8192
+#define PER_CPU_SHARES      1024
+
+typedef struct {
+    ngx_str_t root;
+    ngx_str_t path;
+    ngx_str_t mount_point;
+} cgroup_subsystem_info;
+
+static cgroup_subsystem_info cpu_subsystem;
+
+static ngx_str_t proc_cgroup_file   = ngx_string("/proc/self/cgroup");
+static ngx_str_t proc_mount_file    = ngx_string("/proc/self/mountinfo");
+static ngx_str_t cpu_cfs_period     = ngx_string("/cpu.cfs_period_us");
+static ngx_str_t cpu_cfs_quota      = ngx_string("/cpu.cfs_quota_us");
+static ngx_str_t cpu_cfs_shares     = ngx_string("/cpu.shares");
+
+static float ngx_ceilf(float x)
+{
+    long r = x;
+
+    if (r < 0) {
+        return r;
+    } else {
+        return (r + ((r < x) ? 1 : 0));
+    }
+}
+
+
+static ngx_int_t
+ngx_set_subsystem_path(cgroup_subsystem_info *subsystem_info,
+                       ngx_str_t *cgroup_path, ngx_pool_t *pool)
+{
+    u_char     *p;
+    ngx_uint_t  len;
+
+    if (subsystem_info->root.len != 0 && cgroup_path->data != NULL) {
+        if (ngx_strcmp(subsystem_info->root.data, "/") == 0) {
+            len = subsystem_info->mount_point.len;
+            if (ngx_strcmp(cgroup_path->data, "/") != 0) {
+                len += cgroup_path->len;
+            }
+
+            if (len > NGX_MAX_PATH) {
+                ngx_log_error(NGX_LOG_WARN, pool->log, 0,
+                    "the length of the cgroup path exceeds the maximum " \
+                    "length of the path (%d) ", NGX_MAX_PATH);
+                return NGX_ERROR;
+            }
+
+            p = ngx_palloc(pool, len + 1);
+            if (p == NULL) {
+                return NGX_ERROR;
+            }
+
+            subsystem_info->path.data = p;
+            subsystem_info->path.len = len;
+
+            if (ngx_strcmp(cgroup_path->data, "/") != 0) {
+                p = ngx_sprintf(p, "%V%V",
+                                &subsystem_info->mount_point,
+                                cgroup_path);
+            } else {
+                p = ngx_sprintf(p, "%V", &subsystem_info->mount_point);
+            }
+
+            *p = '\0';
+        } else if (ngx_strcmp(subsystem_info->root.data,
+                              cgroup_path->data) == 0)
+        {
+            subsystem_info->path = subsystem_info->mount_point;
+        }
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_read_proc_file(ngx_str_t *filename, ngx_str_t *buf, ngx_pool_t *pool)
+{
+    ngx_int_t   ret;
+    ngx_uint_t  i;
+    size_t      size;
+    ssize_t     n;
+    ngx_file_t  file;
+
+    ngx_memzero(&file, sizeof(ngx_file_t));
+
+    file.name = *filename;
+    file.log = pool->log;
+    file.fd = ngx_open_file(file.name.data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
+    if (file.fd == NGX_INVALID_FILE) {
+        return NGX_ERROR;
+    }
+
+    ret = NGX_OK;
+
+    //  Typically this file will not be too big, then try to read all data
+    for (i = 1; i <= 5; i++) {
+        size = NGX_BUFFER_SIZE * i;
+        buf->data = ngx_palloc(pool, size);
+        if (buf->data == NULL) {
+            ret = NGX_ERROR;
+            break;
+        }
+
+        n = ngx_read_file(&file, buf->data, size, 0);
+        if (n == NGX_ERROR) {
+            ret = NGX_ERROR;
+            break;
+        }
+
+        buf->len = n;
+
+        if (buf->len < size) {
+            break;
+        }
+    }
+
+    if (ngx_close_file(file.fd) == NGX_FILE_ERROR) {
+        ngx_log_error(NGX_LOG_ALERT, file.log, ngx_errno,
+                      ngx_close_file_n " \"%s\" failed", file.name);
+        return NGX_ERROR;
+    }
+
+    return ret;
+}
+
+
+/*
+ * refers to http://man7.org/linux/man-pages/man7/cgroups.7.html
+ */
+ngx_int_t
+ngx_parse_cgroup_file(ngx_pool_t *pool)
+{
+    ngx_int_t   ret;
+    ngx_str_t   buf;
+    ngx_str_t   fields[3];
+    ngx_uint_t  i, index;
+    u_char     *start, *end, end_of_line;
+
+    ret = ngx_read_proc_file(&proc_cgroup_file, &buf, pool);
+    if (ret == NGX_ERROR) {
+        return ret;
+    }
+
+    index = 0;
+    start = end = buf.data;
+
+    for (i = 0; i < buf.len; i++) {
+        if (*end == ':' || *end == '\n') {
+            end_of_line = (*end == '\n') ? 1 : 0;
+            *end = '\0';
+
+            fields[index].data = start;
+            fields[index].len = end - start;
+
+            if (end_of_line) {
+                index = 0;
+                if (ngx_strstr(fields[1].data, "cpu,cpuacct") != NULL) {
+                    ret = ngx_set_subsystem_path(&cpu_subsystem, &fields[2], pool);
+                    if (ret == NGX_ERROR) {
+                        return ret;
+                    }
+                }
+            } else {
+                index++;
+            }
+
+            start = end + 1;
+        }
+
+        end++;
+    }
+
+    return NGX_OK;
+}
+
+
+/*
+ * refers to http://man7.org/linux/man-pages/man5/proc.5.html
+ */
+ngx_int_t
+ngx_parse_mount_info_file(ngx_pool_t *pool)
+{
+    ngx_str_t   buf;
+    ngx_str_t   fields[11];
+    ngx_int_t   ret;
+    ngx_uint_t  i, index, fs_type_index;
+    u_char     *start, *end, end_of_line;
+
+    ret = ngx_read_proc_file(&proc_mount_file, &buf, pool);
+    if (ret == NGX_ERROR) {
+        return ret;
+    }
+
+    fs_type_index = index = 0;
+    start = end = buf.data;
+
+    for (i = 0; i < buf.len; i++) {
+        if (*end == ' ' || *end == '\n') {
+            end_of_line = (*end == '\n') ? 1 : 0;
+            *end = '\0';
+
+            fields[index].data = start;
+            fields[index].len = end - start;
+
+            if (fields[index].len == 1 && *fields[index].data == '-') {
+                fs_type_index = index + 1;
+            }
+
+            if (end_of_line) {
+                index = 0;
+                if (fields[fs_type_index].len == 6
+                    && ngx_strcmp(fields[fs_type_index].data, "cgroup") == 0
+                    && ngx_strstr(fields[4].data, "cpu,cpuacct") != NULL)
+                {
+                    cpu_subsystem.root = fields[3];
+                    cpu_subsystem.mount_point = fields[4];
+                }
+            } else {
+                index++;
+            }
+
+            start = end + 1;
+        }
+
+        end++;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_read_subsystem_info(cgroup_subsystem_info *subsystem,
+                       ngx_str_t *filename, ngx_log_t *log, ngx_int_t *value)
+{
+    ngx_file_t file;
+    ngx_int_t ret;
+    ngx_str_t full_path;
+    ssize_t   n;
+    u_char   *p;
+    u_char    sign = 0;
+    u_char    buf[NGX_MAX_PATH + 1];
+
+    if (subsystem->path.data == NULL || subsystem->path.len == 0) {
+        return NGX_DONE;
+    }
+
+    if ((subsystem->path.len + filename->len) > NGX_MAX_PATH) {
+        return NGX_DONE;
+    }
+
+    full_path.data = buf;
+    full_path.len = NGX_MAX_PATH;
+
+    ngx_memzero(&file, sizeof(ngx_file_t));
+
+    p = ngx_sprintf(full_path.data, "%V%V", &subsystem->path, filename);
+    *p = '\0';
+    full_path.len = p - full_path.data;
+
+    file.name = full_path;
+    file.log = log;
+    file.fd = ngx_open_file(file.name.data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
+    if (file.fd == NGX_INVALID_FILE) {
+        return NGX_ERROR;
+    }
+
+    n = ngx_read_file(&file, buf, NGX_BUFFER_SIZE, 0);
+    if (n == NGX_ERROR) {
+        return n;
+    }
+
+    if (ngx_close_file(file.fd) == NGX_FILE_ERROR) {
+        ngx_log_error(NGX_LOG_ALERT, file.log, ngx_errno,
+                      ngx_close_file_n " \"%s\" failed", file.name);
+        return NGX_ERROR;
+    }
+
+    n = (n > 0) ? n - 1 : n;    // remove linefeed
+    if (n == 0) {
+        return NGX_DONE;
+    }
+
+    p = buf;
+    if (*p == '-') {
+        sign = 1;
+        p++;
+        n--;
+    }
+
+    ret = ngx_atoi(p, n);
+    if (ret == NGX_ERROR) {
+        return NGX_DONE;
+    }
+
+    if (sign) {
+        *value = -ret;
+    } else {
+        *value = ret;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_get_active_processor_count(ngx_log_t *log) {
+    ngx_int_t       quota_count = 0, share_count = 0;
+    ngx_int_t       cpu_count, limit_count;
+    ngx_int_t       ret, quota, period, shares;
+    ngx_cpuset_t    cpu_affinity;
+
+    ret = ngx_getaffinity(&cpu_affinity, log);
+    if (ret == NGX_ERROR) {
+        return ret;
+    }
+
+    cpu_count = limit_count = ret;
+
+    ngx_log_error(NGX_LOG_INFO, log, 0,
+                  "active processor count: %d", cpu_count);
+
+    ret = ngx_read_subsystem_info(&cpu_subsystem,
+                                     &cpu_cfs_shares, log, &shares);
+    if (ret == NGX_DONE || ret == NGX_ERROR) {
+        return ret;
+    }
+
+    ret = ngx_read_subsystem_info(&cpu_subsystem,
+                                  &cpu_cfs_quota, log, &quota);
+    if (ret == NGX_DONE || ret == NGX_ERROR) {
+        return ret;
+    }
+
+    ret = ngx_read_subsystem_info(&cpu_subsystem,
+                                  &cpu_cfs_period, log, &period);
+    if (ret == NGX_DONE || ret == NGX_ERROR) {
+        return ret;
+    }
+
+    ngx_log_error(NGX_LOG_INFO, log, 0,
+        "cgroup cpu subsystem: quota=%d, period=%d, shares=%d",
+        quota, period, shares);
+
+    if (shares == PER_CPU_SHARES) {
+        shares = -1;
+    }
+
+    if (quota > -1 && period > 0) {
+        quota_count = ngx_ceilf((float) quota / (float) period);
+    }
+
+    if (shares > -1) {
+        share_count = ngx_ceilf((float) shares / (float) PER_CPU_SHARES);
+    }
+
+    if (quota_count != 0 && share_count != 0) {
+        limit_count = ngx_min(quota_count, share_count);
+    } else if (quota_count != 0) {
+        limit_count = quota_count;
+    } else if (share_count != 0) {
+        limit_count = share_count;
+    }
+
+    return ngx_min(cpu_count, limit_count);
+}
+
+
+ngx_int_t
+ngx_container_init(ngx_log_t *log)
+{
+    ngx_pool_t      *pool;
+    ngx_int_t        ret;
+
+    pool = ngx_create_pool(NGX_DEFAULT_POOL_SIZE, log);
+    if (pool == NULL) {
+        return NGX_ERROR;
+    }
+
+    pool->log = log;
+
+    do {
+        ret = ngx_parse_mount_info_file(pool);
+        if (ret != NGX_OK) {
+            break;
+        }
+
+        ret = ngx_parse_cgroup_file(pool);
+        if (ret != NGX_OK) {
+            break;
+        }
+
+        if (cpu_subsystem.path.data == NULL || cpu_subsystem.path.len == 0) {
+            ret = NGX_DONE;
+            break;
+        }
+
+        ret = ngx_get_active_processor_count(log);
+
+    } while (0);
+
+    ngx_destroy_pool(pool);
+
+    return ret;
+}
diff -r 7c614ef3c6ea -r 89793df28d1b src/os/unix/ngx_container.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/os/unix/ngx_container.h	Wed Apr 25 00:05:05 2018 +0800
@@ -0,0 +1,12 @@
+
+/*
+ * Copyright (C) agile6v
+ * Copyright (C) Xiaomi, Inc.
+ */
+
+#ifndef _NGX_CONTAINER_H_INCLUDED_
+#define _NGX_CONTAINER_H_INCLUDED_
+
+ngx_int_t ngx_container_init(ngx_log_t *log);
+
+#endif /* _NGX_CONTAINER_H_INCLUDED_ */
diff -r 7c614ef3c6ea -r 89793df28d1b src/os/unix/ngx_linux_init.c
--- a/src/os/unix/ngx_linux_init.c	Wed Apr 18 16:11:41 2018 +0300
+++ b/src/os/unix/ngx_linux_init.c	Wed Apr 25 00:05:05 2018 +0800
@@ -7,12 +7,12 @@
 
 #include <ngx_config.h>
 #include <ngx_core.h>
+#include <ngx_container.h>
 
 
 u_char  ngx_linux_kern_ostype[50];
 u_char  ngx_linux_kern_osrelease[50];
 
-
 static ngx_os_io_t ngx_linux_io = {
     ngx_unix_recv,
     ngx_readv_chain,
@@ -33,6 +33,7 @@
 ngx_int_t
 ngx_os_specific_init(ngx_log_t *log)
 {
+    ngx_int_t       ret;
     struct utsname  u;
 
     if (uname(&u) == -1) {
@@ -48,6 +49,13 @@
 
     ngx_os_io = ngx_linux_io;
 
+    ret = ngx_container_init(log);
+    if (ret == NGX_ERROR) {
+        return ret;
+    } else if (ret > NGX_OK) {
+        ngx_ncpu = ret;
+    }
+
     return NGX_OK;
 }
 
diff -r 7c614ef3c6ea -r 89793df28d1b src/os/unix/ngx_setaffinity.c
--- a/src/os/unix/ngx_setaffinity.c	Wed Apr 18 16:11:41 2018 +0300
+++ b/src/os/unix/ngx_setaffinity.c	Wed Apr 25 00:05:05 2018 +0800
@@ -30,6 +30,12 @@
     }
 }
 
+ngx_int_t
+ngx_getaffinity(ngx_cpuset_t *cpu_affinity, ngx_log_t *log)
+{
+    return 0;
+}
+
 #elif (NGX_HAVE_SCHED_SETAFFINITY)
 
 void
@@ -50,4 +56,16 @@
     }
 }
 
+ngx_int_t
+ngx_getaffinity(ngx_cpuset_t *cpu_affinity, ngx_log_t *log)
+{
+    if (sched_getaffinity(0, sizeof(ngx_cpuset_t), cpu_affinity) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
+                      "sched_getaffinity() failed");
+        return NGX_ERROR;
+    }
+
+    return CPU_COUNT(cpu_affinity);
+}
+
 #endif
diff -r 7c614ef3c6ea -r 89793df28d1b src/os/unix/ngx_setaffinity.h
--- a/src/os/unix/ngx_setaffinity.h	Wed Apr 18 16:11:41 2018 +0300
+++ b/src/os/unix/ngx_setaffinity.h	Wed Apr 25 00:05:05 2018 +0800
@@ -23,12 +23,15 @@
 
 #endif
 
+ngx_int_t ngx_getaffinity(ngx_cpuset_t *cpu_affinity, ngx_log_t *log);
 void ngx_setaffinity(ngx_cpuset_t *cpu_affinity, ngx_log_t *log);
 
 #else
 
 #define ngx_setaffinity(cpu_affinity, log)
 
+#define ngx_getaffinity(cpu_affinity, log)
+
 typedef uint64_t  ngx_cpuset_t;
 
 #endif
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20180425/78b1e2e4/attachment-0001.html>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: nginx.patch
Type: application/octet-stream
Size: 14579 bytes
Desc: not available
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20180425/78b1e2e4/attachment-0001.obj>


More information about the nginx-devel mailing list