[nginx] Generic subrequests in memory.
Roman Arutyunyan
arut at nginx.com
Wed Feb 28 15:01:43 UTC 2018
details: http://hg.nginx.org/nginx/rev/20f139e9ffa8
branches:
changeset: 7220:20f139e9ffa8
user: Roman Arutyunyan <arut at nginx.com>
date: Wed Feb 28 16:56:58 2018 +0300
description:
Generic subrequests in memory.
Previously, only the upstream response body could be accessed with the
NGX_HTTP_SUBREQUEST_IN_MEMORY feature. Now any response body from a subrequest
can be saved in a memory buffer. It is available as a single buffer in r->out
and the buffer size is configured by the subrequest_output_buffer_size
directive.
Upstream, proxy and fastcgi code used to handle the old-style feature is
removed.
diffstat:
src/http/modules/ngx_http_fastcgi_module.c | 30 ------
src/http/modules/ngx_http_proxy_module.c | 30 ------
src/http/modules/ngx_http_ssi_filter_module.c | 8 +-
src/http/ngx_http_core_module.c | 21 ++++
src/http/ngx_http_core_module.h | 2 +
src/http/ngx_http_postpone_filter_module.c | 78 ++++++++++++++++
src/http/ngx_http_upstream.c | 126 +-------------------------
7 files changed, 107 insertions(+), 188 deletions(-)
diffs (428 lines):
diff -r d0d32b33167d -r 20f139e9ffa8 src/http/modules/ngx_http_fastcgi_module.c
--- a/src/http/modules/ngx_http_fastcgi_module.c Thu Feb 22 17:25:43 2018 +0300
+++ b/src/http/modules/ngx_http_fastcgi_module.c Wed Feb 28 16:56:58 2018 +0300
@@ -2512,36 +2512,6 @@ ngx_http_fastcgi_non_buffered_filter(voi
break;
}
- /* provide continuous buffer for subrequests in memory */
-
- if (r->subrequest_in_memory) {
-
- cl = u->out_bufs;
-
- if (cl) {
- buf->pos = cl->buf->pos;
- }
-
- buf->last = buf->pos;
-
- for (cl = u->out_bufs; cl; cl = cl->next) {
- ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "http fastcgi in memory %p-%p %O",
- cl->buf->pos, cl->buf->last, ngx_buf_size(cl->buf));
-
- if (buf->last == cl->buf->pos) {
- buf->last = cl->buf->last;
- continue;
- }
-
- buf->last = ngx_movemem(buf->last, cl->buf->pos,
- cl->buf->last - cl->buf->pos);
-
- cl->buf->pos = buf->last - (cl->buf->last - cl->buf->pos);
- cl->buf->last = buf->last;
- }
- }
-
return NGX_OK;
}
diff -r d0d32b33167d -r 20f139e9ffa8 src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c Thu Feb 22 17:25:43 2018 +0300
+++ b/src/http/modules/ngx_http_proxy_module.c Wed Feb 28 16:56:58 2018 +0300
@@ -2321,36 +2321,6 @@ ngx_http_proxy_non_buffered_chunked_filt
return NGX_ERROR;
}
- /* provide continuous buffer for subrequests in memory */
-
- if (r->subrequest_in_memory) {
-
- cl = u->out_bufs;
-
- if (cl) {
- buf->pos = cl->buf->pos;
- }
-
- buf->last = buf->pos;
-
- for (cl = u->out_bufs; cl; cl = cl->next) {
- ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "http proxy in memory %p-%p %O",
- cl->buf->pos, cl->buf->last, ngx_buf_size(cl->buf));
-
- if (buf->last == cl->buf->pos) {
- buf->last = cl->buf->last;
- continue;
- }
-
- buf->last = ngx_movemem(buf->last, cl->buf->pos,
- cl->buf->last - cl->buf->pos);
-
- cl->buf->pos = buf->last - (cl->buf->last - cl->buf->pos);
- cl->buf->last = buf->last;
- }
- }
-
return NGX_OK;
}
diff -r d0d32b33167d -r 20f139e9ffa8 src/http/modules/ngx_http_ssi_filter_module.c
--- a/src/http/modules/ngx_http_ssi_filter_module.c Thu Feb 22 17:25:43 2018 +0300
+++ b/src/http/modules/ngx_http_ssi_filter_module.c Wed Feb 28 16:56:58 2018 +0300
@@ -2231,9 +2231,11 @@ ngx_http_ssi_set_variable(ngx_http_reque
{
ngx_str_t *value = data;
- if (r->upstream) {
- value->len = r->upstream->buffer.last - r->upstream->buffer.pos;
- value->data = r->upstream->buffer.pos;
+ if (r->headers_out.status < NGX_HTTP_SPECIAL_RESPONSE
+ && r->out && r->out->buf)
+ {
+ value->len = r->out->buf->last - r->out->buf->pos;
+ value->data = r->out->buf->pos;
}
return rc;
diff -r d0d32b33167d -r 20f139e9ffa8 src/http/ngx_http_core_module.c
--- a/src/http/ngx_http_core_module.c Thu Feb 22 17:25:43 2018 +0300
+++ b/src/http/ngx_http_core_module.c Wed Feb 28 16:56:58 2018 +0300
@@ -399,6 +399,13 @@ static ngx_command_t ngx_http_core_comm
offsetof(ngx_http_core_loc_conf_t, sendfile_max_chunk),
NULL },
+ { ngx_string("subrequest_output_buffer_size"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_size_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_core_loc_conf_t, subrequest_output_buffer_size),
+ NULL },
+
{ ngx_string("aio"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
ngx_http_core_set_aio,
@@ -2237,6 +2244,12 @@ ngx_http_subrequest(ngx_http_request_t *
return NGX_ERROR;
}
+ if (r->subrequest_in_memory) {
+ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+ "nested in-memory subrequest \"%V\"", uri);
+ return NGX_ERROR;
+ }
+
sr = ngx_pcalloc(r->pool, sizeof(ngx_http_request_t));
if (sr == NULL) {
return NGX_ERROR;
@@ -2318,6 +2331,10 @@ ngx_http_subrequest(ngx_http_request_t *
sr->log_handler = r->log_handler;
+ if (sr->subrequest_in_memory) {
+ sr->filter_need_in_memory = 1;
+ }
+
if (!sr->background) {
if (c->data == r && r->postponed == NULL) {
c->data = sr;
@@ -3356,6 +3373,7 @@ ngx_http_core_create_loc_conf(ngx_conf_t
clcf->internal = NGX_CONF_UNSET;
clcf->sendfile = NGX_CONF_UNSET;
clcf->sendfile_max_chunk = NGX_CONF_UNSET_SIZE;
+ clcf->subrequest_output_buffer_size = NGX_CONF_UNSET_SIZE;
clcf->aio = NGX_CONF_UNSET;
clcf->aio_write = NGX_CONF_UNSET;
#if (NGX_THREADS)
@@ -3578,6 +3596,9 @@ ngx_http_core_merge_loc_conf(ngx_conf_t
ngx_conf_merge_value(conf->sendfile, prev->sendfile, 0);
ngx_conf_merge_size_value(conf->sendfile_max_chunk,
prev->sendfile_max_chunk, 0);
+ ngx_conf_merge_size_value(conf->subrequest_output_buffer_size,
+ prev->subrequest_output_buffer_size,
+ (size_t) ngx_pagesize);
ngx_conf_merge_value(conf->aio, prev->aio, NGX_HTTP_AIO_OFF);
ngx_conf_merge_value(conf->aio_write, prev->aio_write, 0);
#if (NGX_THREADS)
diff -r d0d32b33167d -r 20f139e9ffa8 src/http/ngx_http_core_module.h
--- a/src/http/ngx_http_core_module.h Thu Feb 22 17:25:43 2018 +0300
+++ b/src/http/ngx_http_core_module.h Wed Feb 28 16:56:58 2018 +0300
@@ -351,6 +351,8 @@ struct ngx_http_core_loc_conf_s {
size_t limit_rate_after; /* limit_rate_after */
size_t sendfile_max_chunk; /* sendfile_max_chunk */
size_t read_ahead; /* read_ahead */
+ size_t subrequest_output_buffer_size;
+ /* subrequest_output_buffer_size */
ngx_msec_t client_body_timeout; /* client_body_timeout */
ngx_msec_t send_timeout; /* send_timeout */
diff -r d0d32b33167d -r 20f139e9ffa8 src/http/ngx_http_postpone_filter_module.c
--- a/src/http/ngx_http_postpone_filter_module.c Thu Feb 22 17:25:43 2018 +0300
+++ b/src/http/ngx_http_postpone_filter_module.c Wed Feb 28 16:56:58 2018 +0300
@@ -12,6 +12,8 @@
static ngx_int_t ngx_http_postpone_filter_add(ngx_http_request_t *r,
ngx_chain_t *in);
+static ngx_int_t ngx_http_postpone_filter_in_memory(ngx_http_request_t *r,
+ ngx_chain_t *in);
static ngx_int_t ngx_http_postpone_filter_init(ngx_conf_t *cf);
@@ -60,6 +62,10 @@ ngx_http_postpone_filter(ngx_http_reques
ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0,
"http postpone filter \"%V?%V\" %p", &r->uri, &r->args, in);
+ if (r->subrequest_in_memory) {
+ return ngx_http_postpone_filter_in_memory(r, in);
+ }
+
if (r != c->data) {
if (in) {
@@ -172,6 +178,78 @@ found:
static ngx_int_t
+ngx_http_postpone_filter_in_memory(ngx_http_request_t *r, ngx_chain_t *in)
+{
+ size_t len;
+ ngx_buf_t *b;
+ ngx_connection_t *c;
+ ngx_http_core_loc_conf_t *clcf;
+
+ c = r->connection;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
+ "http postpone filter in memory");
+
+ if (r->out == NULL) {
+ r->out = ngx_alloc_chain_link(r->pool);
+ if (r->out == NULL) {
+ return NGX_ERROR;
+ }
+
+ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
+ if (r->headers_out.content_length_n != -1) {
+ len = r->headers_out.content_length_n;
+
+ if (len > clcf->subrequest_output_buffer_size) {
+ ngx_log_error(NGX_LOG_ERR, c->log, 0,
+ "too big subrequest response: %uz", len);
+ return NGX_ERROR;
+ }
+
+ } else {
+ len = clcf->subrequest_output_buffer_size;
+ }
+
+ b = ngx_create_temp_buf(r->pool, len);
+ if (b == NULL) {
+ return NGX_ERROR;
+ }
+
+ b->last_buf = 1;
+
+ r->out->buf = b;
+ r->out->next = NULL;
+ }
+
+ b = r->out->buf;
+
+ for ( /* void */ ; in; in = in->next) {
+
+ if (ngx_buf_special(in->buf)) {
+ continue;
+ }
+
+ len = in->buf->last - in->buf->pos;
+
+ if (len > (size_t) (b->end - b->last)) {
+ ngx_log_error(NGX_LOG_ERR, c->log, 0,
+ "too big subrequest response");
+ return NGX_ERROR;
+ }
+
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
+ "http postpone filter in memory %uz bytes", len);
+
+ b->last = ngx_cpymem(b->last, in->buf->pos, len);
+ in->buf->pos = in->buf->last;
+ }
+
+ return NGX_OK;
+}
+
+
+static ngx_int_t
ngx_http_postpone_filter_init(ngx_conf_t *cf)
{
ngx_http_next_body_filter = ngx_http_top_body_filter;
diff -r d0d32b33167d -r 20f139e9ffa8 src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c Thu Feb 22 17:25:43 2018 +0300
+++ b/src/http/ngx_http_upstream.c Wed Feb 28 16:56:58 2018 +0300
@@ -55,8 +55,6 @@ static ngx_int_t ngx_http_upstream_inter
static ngx_int_t ngx_http_upstream_test_connect(ngx_connection_t *c);
static ngx_int_t ngx_http_upstream_process_headers(ngx_http_request_t *r,
ngx_http_upstream_t *u);
-static void ngx_http_upstream_process_body_in_memory(ngx_http_request_t *r,
- ngx_http_upstream_t *u);
static void ngx_http_upstream_send_response(ngx_http_request_t *r,
ngx_http_upstream_t *u);
static void ngx_http_upstream_upgrade(ngx_http_request_t *r,
@@ -2335,45 +2333,7 @@ ngx_http_upstream_process_header(ngx_htt
return;
}
- if (!r->subrequest_in_memory) {
- ngx_http_upstream_send_response(r, u);
- return;
- }
-
- /* subrequest content in memory */
-
- if (u->input_filter == NULL) {
- u->input_filter_init = ngx_http_upstream_non_buffered_filter_init;
- u->input_filter = ngx_http_upstream_non_buffered_filter;
- u->input_filter_ctx = r;
- }
-
- if (u->input_filter_init(u->input_filter_ctx) == NGX_ERROR) {
- ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
- return;
- }
-
- n = u->buffer.last - u->buffer.pos;
-
- if (n) {
- u->buffer.last = u->buffer.pos;
-
- u->state->response_length += n;
-
- if (u->input_filter(u->input_filter_ctx, n) == NGX_ERROR) {
- ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
- return;
- }
- }
-
- if (u->length == 0) {
- ngx_http_upstream_finalize_request(r, u, 0);
- return;
- }
-
- u->read_event_handler = ngx_http_upstream_process_body_in_memory;
-
- ngx_http_upstream_process_body_in_memory(r, u);
+ ngx_http_upstream_send_response(r, u);
}
@@ -2776,84 +2736,6 @@ ngx_http_upstream_process_headers(ngx_ht
static void
-ngx_http_upstream_process_body_in_memory(ngx_http_request_t *r,
- ngx_http_upstream_t *u)
-{
- size_t size;
- ssize_t n;
- ngx_buf_t *b;
- ngx_event_t *rev;
- ngx_connection_t *c;
-
- c = u->peer.connection;
- rev = c->read;
-
- ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
- "http upstream process body in memory");
-
- if (rev->timedout) {
- ngx_connection_error(c, NGX_ETIMEDOUT, "upstream timed out");
- ngx_http_upstream_finalize_request(r, u, NGX_HTTP_GATEWAY_TIME_OUT);
- return;
- }
-
- b = &u->buffer;
-
- for ( ;; ) {
-
- size = b->end - b->last;
-
- if (size == 0) {
- ngx_log_error(NGX_LOG_ALERT, c->log, 0,
- "upstream buffer is too small to read response");
- ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
- return;
- }
-
- n = c->recv(c, b->last, size);
-
- if (n == NGX_AGAIN) {
- break;
- }
-
- if (n == 0 || n == NGX_ERROR) {
- ngx_http_upstream_finalize_request(r, u, n);
- return;
- }
-
- u->state->bytes_received += n;
- u->state->response_length += n;
-
- if (u->input_filter(u->input_filter_ctx, n) == NGX_ERROR) {
- ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
- return;
- }
-
- if (!rev->ready) {
- break;
- }
- }
-
- if (u->length == 0) {
- ngx_http_upstream_finalize_request(r, u, 0);
- return;
- }
-
- if (ngx_handle_read_event(rev, 0) != NGX_OK) {
- ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
- return;
- }
-
- if (rev->active) {
- ngx_add_timer(rev, u->conf->read_timeout);
-
- } else if (rev->timer_set) {
- ngx_del_timer(rev);
- }
-}
-
-
-static void
ngx_http_upstream_send_response(ngx_http_request_t *r, ngx_http_upstream_t *u)
{
ssize_t n;
@@ -4359,12 +4241,6 @@ ngx_http_upstream_finalize_request(ngx_h
#endif
- if (r->subrequest_in_memory
- && u->headers_in.status_n >= NGX_HTTP_SPECIAL_RESPONSE)
- {
- u->buffer.last = u->buffer.pos;
- }
-
r->read_event_handler = ngx_http_block_reading;
if (rc == NGX_DECLINED) {
More information about the nginx-devel
mailing list