[PATCH] introduce new variable fs_watermark for proxy_cache_path
Adam Bambuch
adam.bambuch2 at gmail.com
Wed May 6 23:40:54 UTC 2020
# HG changeset patch
# User Adam Bambuch <adam.bambuch2 at gmail.com>
# Date 1588808163 -7200
# Thu May 07 01:36:03 2020 +0200
# Node ID 8d054b64f07457cad2b74376d5f88162c887ba35
# Parent 716eddd74bc2831537f5b3f7ecd16ad3e516d043
introduce new variable fs_watermark for proxy_cache_path
This configuration parameter should help with better disk usage,
especially in environments, where nginx doesn't have a separate
partition for caching and disk space is shared between multiple apps.
It could also help in environments, where cache contains many small
files, and cache loader can't load all the files in a reasonable time
and therefore max_size is exceeded for many hours after nginx restart.
diff -r 716eddd74bc2 -r 8d054b64f074 src/http/ngx_http_cache.h
--- a/src/http/ngx_http_cache.h Thu Apr 23 15:10:26 2020 +0300
+++ b/src/http/ngx_http_cache.h Thu May 07 01:36:03 2020 +0200
@@ -160,6 +160,7 @@
ngx_path_t *path;
+ off_t fs_watermark;
off_t max_size;
size_t bsize;
diff -r 716eddd74bc2 -r 8d054b64f074 src/http/ngx_http_file_cache.c
--- a/src/http/ngx_http_file_cache.c Thu Apr 23 15:10:26 2020 +0300
+++ b/src/http/ngx_http_file_cache.c Thu May 07 01:36:03 2020 +0200
@@ -1959,7 +1959,7 @@
{
ngx_http_file_cache_t *cache = data;
- off_t size;
+ off_t fs_available, size;
time_t wait;
ngx_msec_t elapsed, next;
ngx_uint_t count, watermark;
@@ -1983,11 +1983,22 @@
ngx_shmtx_unlock(&cache->shpool->mutex);
- ngx_log_debug3(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
- "http file cache size: %O c:%ui w:%i",
- size, count, (ngx_int_t) watermark);
-
- if (size < cache->max_size && count < watermark) {
+ fs_available = NGX_MAX_OFF_T_VALUE;
+ if (cache->fs_watermark) {
+ fs_available = ngx_fs_available(cache->path->name.data);
+ if (fs_available == NGX_ERROR) {
+ ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno,
+ ngx_fs_available_n " \"%s\" failed",
cache->path->name.data);
+ }
+ }
+
+ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
+ "http file cache size: %O c:%ui w:%i fa:%O",
+ size, count, (ngx_int_t) watermark, fs_available);
+
+ if (size < cache->max_size &&
+ count < watermark &&
+ fs_available >= cache->fs_watermark) {
break;
}
@@ -2304,7 +2315,7 @@
{
char *confp = conf;
- off_t max_size;
+ off_t fs_watermark, max_size;
u_char *last, *p;
time_t inactive;
ssize_t size;
@@ -2340,6 +2351,7 @@
name.len = 0;
size = 0;
+ fs_watermark = 0;
max_size = NGX_MAX_OFF_T_VALUE;
value = cf->args->elts;
@@ -2461,6 +2473,26 @@
continue;
}
+ #if (NGX_WIN32 || NGX_HAVE_STATFS || NGX_HAVE_STATVFS)
+
+ if (ngx_strncmp(value[i].data, "fs_watermark=", 13) == 0) {
+
+ s.len = value[i].len - 13;
+ s.data = value[i].data + 13;
+
+ fs_watermark = ngx_parse_offset(&s);
+ if (fs_watermark < 0) {
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "invalid fs_watermark value \"%V\"",
+ &value[i]);
+ return NGX_CONF_ERROR;
+ }
+
+ continue;
+ }
+
+ #endif
+
if (ngx_strncmp(value[i].data, "max_size=", 9) == 0) {
s.len = value[i].len - 9;
@@ -2606,6 +2638,7 @@
cache->use_temp_path = use_temp_path;
cache->inactive = inactive;
+ cache->fs_watermark = fs_watermark;
cache->max_size = max_size;
caches = (ngx_array_t *) (confp + cmd->offset);
diff -r 716eddd74bc2 -r 8d054b64f074 src/os/unix/ngx_files.c
--- a/src/os/unix/ngx_files.c Thu Apr 23 15:10:26 2020 +0300
+++ b/src/os/unix/ngx_files.c Thu May 07 01:36:03 2020 +0200
@@ -878,6 +878,18 @@
return (size_t) fs.f_bsize;
}
+off_t
+ngx_fs_available(u_char *name)
+{
+ struct statfs fs;
+
+ if (statfs((char *) name, &fs) == -1) {
+ return NGX_ERROR;
+ }
+
+ return (ssize_t) (fs.f_bavail * fs.f_bsize);
+}
+
#elif (NGX_HAVE_STATVFS)
size_t
@@ -896,6 +908,18 @@
return (size_t) fs.f_frsize;
}
+off_t
+ngx_fs_available(u_char *name)
+{
+ struct statvfs fs;
+
+ if (statvfs((char *) name, &fs) == -1) {
+ return NGX_ERROR;
+ }
+
+ return (ssize_t) (fs.f_bavail * fs.f_bsize);
+}
+
#else
size_t
@@ -904,4 +928,10 @@
return 512;
}
+off_t
+ngx_fs_available(u_char *name)
+{
+ return NGX_ERROR;
+}
+
#endif
diff -r 716eddd74bc2 -r 8d054b64f074 src/os/unix/ngx_files.h
--- a/src/os/unix/ngx_files.h Thu Apr 23 15:10:26 2020 +0300
+++ b/src/os/unix/ngx_files.h Thu May 07 01:36:03 2020 +0200
@@ -347,6 +347,23 @@
size_t ngx_fs_bsize(u_char *name);
+off_t ngx_fs_available(u_char *name);
+
+
+#if (NGX_HAVE_STATFS)
+
+#define ngx_fs_available_n "statfs()"
+
+#elif (NGX_HAVE_STATVFS)
+
+#define ngx_fs_available_n "statvfs()"
+
+#else
+
+#define ngx_fs_available_n "ngx_fs_available()"
+
+#endif
+
#if (NGX_HAVE_OPENAT)
diff -r 716eddd74bc2 -r 8d054b64f074 src/os/win32/ngx_files.c
--- a/src/os/win32/ngx_files.c Thu Apr 23 15:10:26 2020 +0300
+++ b/src/os/win32/ngx_files.c Thu May 07 01:36:03 2020 +0200
@@ -657,6 +657,18 @@
return sc * bs;
}
+off_t
+ngx_fs_available(u_char *name)
+{
+ ULARGE_INTEGER navail, ntotal, nfree;
+
+ if (GetDiskFreeSpaceEx((const char *) name, &navail, &ntotal,
&nfree) == 0) {
+ return NGX_ERROR;
+ }
+
+ return (off_t) navail.QuadPart;
+}
+
static ngx_int_t
ngx_win32_check_filename(u_char *name, u_short *u, size_t len)
diff -r 716eddd74bc2 -r 8d054b64f074 src/os/win32/ngx_files.h
--- a/src/os/win32/ngx_files.h Thu Apr 23 15:10:26 2020 +0300
+++ b/src/os/win32/ngx_files.h Thu May 07 01:36:03 2020 +0200
@@ -260,6 +260,9 @@
size_t ngx_fs_bsize(u_char *name);
+off_t ngx_fs_available(u_char *name);
+#define ngx_fs_available_n "GetDiskFreeSpaceEx"
+
#define ngx_stdout GetStdHandle(STD_OUTPUT_HANDLE)
#define ngx_stderr GetStdHandle(STD_ERROR_HANDLE)
More information about the nginx-devel
mailing list