[nginx] HTTP/2: fixed "task already active" with sendfile in threads.
Maxim Dounin
mdounin at mdounin.ru
Thu Nov 25 19:03:46 UTC 2021
details: https://hg.nginx.org/nginx/rev/555533169506
branches:
changeset: 7974:555533169506
user: Maxim Dounin <mdounin at mdounin.ru>
date: Thu Nov 25 22:02:05 2021 +0300
description:
HTTP/2: fixed "task already active" with sendfile in threads.
With sendfile in threads, "task already active" alerts might appear in logs
if a write event happens on the main HTTP/2 connection, triggering a sendfile
in threads while another thread operation is already running. Observed
with "aio threads; aio_write on; sendfile on;" and with thread event handlers
modified to post a write event to the main HTTP/2 connection (though can
happen without any modifications).
Similarly, sendfile() with AIO preloading on FreeBSD can trigger duplicate
aio operation, resulting in "second aio post" alerts. This is, however,
harder to reproduce, especially on modern FreeBSD systems, since sendfile()
usually does not return EBUSY.
Fix is to avoid starting a sendfile operation if other thread operation
is active by checking r->aio in the thread handler (and, similarly, in
aio preload handler). The added check also makes duplicate calls protection
redundant, so it is removed.
diffstat:
src/http/ngx_http_copy_filter_module.c | 38 ++++++++++++++++++++++++++++++-
src/http/ngx_http_upstream.c | 22 ++++++++++++++++++
src/os/unix/ngx_freebsd_sendfile_chain.c | 13 ----------
src/os/unix/ngx_linux_sendfile_chain.c | 9 -------
4 files changed, 58 insertions(+), 24 deletions(-)
diffs (148 lines):
diff -r 3443c02ca1d1 -r 555533169506 src/http/ngx_http_copy_filter_module.c
--- a/src/http/ngx_http_copy_filter_module.c Mon Nov 01 18:09:34 2021 +0300
+++ b/src/http/ngx_http_copy_filter_module.c Thu Nov 25 22:02:05 2021 +0300
@@ -219,13 +219,25 @@ ngx_http_copy_aio_sendfile_preload(ngx_b
ngx_http_request_t *r;
ngx_output_chain_ctx_t *ctx;
+ aio = file->file->aio;
+ r = aio->data;
+
+ if (r->aio) {
+ /*
+ * tolerate sendfile() calls if another operation is already
+ * running; this can happen due to subrequests, multiple calls
+ * of the next body filter from a filter, or in HTTP/2 due to
+ * a write event on the main connection
+ */
+
+ return NGX_AGAIN;
+ }
+
n = ngx_file_aio_read(file->file, buf, 1, file->file_pos, NULL);
if (n == NGX_AGAIN) {
- aio = file->file->aio;
aio->handler = ngx_http_copy_aio_sendfile_event_handler;
- r = aio->data;
r->main->blocked++;
r->aio = 1;
@@ -263,6 +275,7 @@ static ngx_int_t
ngx_http_copy_thread_handler(ngx_thread_task_t *task, ngx_file_t *file)
{
ngx_str_t name;
+ ngx_connection_t *c;
ngx_thread_pool_t *tp;
ngx_http_request_t *r;
ngx_output_chain_ctx_t *ctx;
@@ -270,6 +283,27 @@ ngx_http_copy_thread_handler(ngx_thread_
r = file->thread_ctx;
+ if (r->aio) {
+ /*
+ * tolerate sendfile() calls if another operation is already
+ * running; this can happen due to subrequests, multiple calls
+ * of the next body filter from a filter, or in HTTP/2 due to
+ * a write event on the main connection
+ */
+
+ c = r->connection;
+
+#if (NGX_HTTP_V2)
+ if (r->stream) {
+ c = r->stream->connection->connection;
+ }
+#endif
+
+ if (task == c->sendfile_task) {
+ return NGX_OK;
+ }
+ }
+
clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
tp = clcf->thread_pool;
diff -r 3443c02ca1d1 -r 555533169506 src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c Mon Nov 01 18:09:34 2021 +0300
+++ b/src/http/ngx_http_upstream.c Thu Nov 25 22:02:05 2021 +0300
@@ -3847,6 +3847,7 @@ ngx_http_upstream_thread_handler(ngx_thr
{
ngx_str_t name;
ngx_event_pipe_t *p;
+ ngx_connection_t *c;
ngx_thread_pool_t *tp;
ngx_http_request_t *r;
ngx_http_core_loc_conf_t *clcf;
@@ -3854,6 +3855,27 @@ ngx_http_upstream_thread_handler(ngx_thr
r = file->thread_ctx;
p = r->upstream->pipe;
+ if (r->aio) {
+ /*
+ * tolerate sendfile() calls if another operation is already
+ * running; this can happen due to subrequests, multiple calls
+ * of the next body filter from a filter, or in HTTP/2 due to
+ * a write event on the main connection
+ */
+
+ c = r->connection;
+
+#if (NGX_HTTP_V2)
+ if (r->stream) {
+ c = r->stream->connection->connection;
+ }
+#endif
+
+ if (task == c->sendfile_task) {
+ return NGX_OK;
+ }
+ }
+
clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
tp = clcf->thread_pool;
diff -r 3443c02ca1d1 -r 555533169506 src/os/unix/ngx_freebsd_sendfile_chain.c
--- a/src/os/unix/ngx_freebsd_sendfile_chain.c Mon Nov 01 18:09:34 2021 +0300
+++ b/src/os/unix/ngx_freebsd_sendfile_chain.c Thu Nov 25 22:02:05 2021 +0300
@@ -255,19 +255,6 @@ ngx_freebsd_sendfile_chain(ngx_connectio
#if (NGX_HAVE_AIO_SENDFILE)
if (ebusy) {
- if (aio->event.active) {
- /*
- * tolerate duplicate calls; they can happen due to subrequests
- * or multiple calls of the next body filter from a filter
- */
-
- if (sent) {
- c->busy_count = 0;
- }
-
- return in;
- }
-
if (sent == 0) {
c->busy_count++;
diff -r 3443c02ca1d1 -r 555533169506 src/os/unix/ngx_linux_sendfile_chain.c
--- a/src/os/unix/ngx_linux_sendfile_chain.c Mon Nov 01 18:09:34 2021 +0300
+++ b/src/os/unix/ngx_linux_sendfile_chain.c Thu Nov 25 22:02:05 2021 +0300
@@ -379,15 +379,6 @@ ngx_linux_sendfile_thread(ngx_connection
return ctx->sent;
}
- if (task->event.active && ctx->file == file) {
- /*
- * tolerate duplicate calls; they can happen due to subrequests
- * or multiple calls of the next body filter from a filter
- */
-
- return NGX_DONE;
- }
-
ctx->file = file;
ctx->socket = c->fd;
ctx->size = size;
More information about the nginx-devel
mailing list