[nginx] Slice filter.
Roman Arutyunyan
arut at nginx.com
Mon Dec 7 14:14:53 UTC 2015
details: http://hg.nginx.org/nginx/rev/29f35e60840b
branches:
changeset: 6317:29f35e60840b
user: Roman Arutyunyan <arut at nginx.com>
date: Mon Dec 07 16:30:48 2015 +0300
description:
Slice filter.
Splits a request into subrequests, each providing a specific range of response.
The variable "$slice_range" must be used to set subrequest range and proper
cache key. The directive "slice" sets slice size.
The following example splits requests into 1-megabyte cacheable subrequests.
server {
listen 8000;
location / {
slice 1m;
proxy_cache cache;
proxy_cache_key $uri$is_args$args$slice_range;
proxy_set_header Range $slice_range;
proxy_cache_valid 200 206 1h;
proxy_pass http://127.0.0.1:9000;
}
}
diffstat:
auto/modules | 15 +-
auto/options | 3 +
auto/sources | 4 +
src/http/modules/ngx_http_range_filter_module.c | 28 +-
src/http/modules/ngx_http_slice_filter_module.c | 521 ++++++++++++++++++++++++
src/http/ngx_http_request.h | 2 +
6 files changed, 567 insertions(+), 6 deletions(-)
diffs (truncated from 697 to 300 lines):
diff -r f44de0d12143 -r 29f35e60840b auto/modules
--- a/auto/modules Mon Dec 07 16:30:47 2015 +0300
+++ b/auto/modules Mon Dec 07 16:30:48 2015 +0300
@@ -73,6 +73,11 @@ if [ $HTTP_SSI = YES ]; then
fi
+if [ $HTTP_SLICE = YES ]; then
+ HTTP_POSTPONE=YES
+fi
+
+
if [ $HTTP_ADDITION = YES ]; then
HTTP_POSTPONE=YES
fi
@@ -110,6 +115,7 @@ fi
# ngx_http_copy_filter
# ngx_http_range_body_filter
# ngx_http_not_modified_filter
+# ngx_http_slice_filter
HTTP_FILTER_MODULES="$HTTP_WRITE_FILTER_MODULE \
$HTTP_HEADER_FILTER_MODULE \
@@ -179,6 +185,12 @@ if [ $HTTP_USERID = YES ]; then
HTTP_SRCS="$HTTP_SRCS $HTTP_USERID_SRCS"
fi
+if [ $HTTP_SLICE = YES ]; then
+ HTTP_SRCS="$HTTP_SRCS $HTTP_SLICE_SRCS"
+else
+ HTTP_SLICE_FILTER_MODULE=""
+fi
+
if [ $HTTP_V2 = YES ]; then
have=NGX_HTTP_V2 . auto/have
@@ -461,7 +473,8 @@ if [ $HTTP = YES ]; then
$HTTP_AUX_FILTER_MODULES \
$HTTP_COPY_FILTER_MODULE \
$HTTP_RANGE_BODY_FILTER_MODULE \
- $HTTP_NOT_MODIFIED_FILTER_MODULE"
+ $HTTP_NOT_MODIFIED_FILTER_MODULE \
+ $HTTP_SLICE_FILTER_MODULE"
NGX_ADDON_DEPS="$NGX_ADDON_DEPS \$(HTTP_DEPS)"
fi
diff -r f44de0d12143 -r 29f35e60840b auto/options
--- a/auto/options Mon Dec 07 16:30:47 2015 +0300
+++ b/auto/options Mon Dec 07 16:30:48 2015 +0300
@@ -71,6 +71,7 @@ HTTP_ACCESS=YES
HTTP_AUTH_BASIC=YES
HTTP_AUTH_REQUEST=NO
HTTP_USERID=YES
+HTTP_SLICE=NO
HTTP_AUTOINDEX=YES
HTTP_RANDOM_INDEX=NO
HTTP_STATUS=NO
@@ -226,6 +227,7 @@ do
--with-http_random_index_module) HTTP_RANDOM_INDEX=YES ;;
--with-http_secure_link_module) HTTP_SECURE_LINK=YES ;;
--with-http_degradation_module) HTTP_DEGRADATION=YES ;;
+ --with-http_slice_module) HTTP_SLICE=YES ;;
--without-http_charset_module) HTTP_CHARSET=NO ;;
--without-http_gzip_module) HTTP_GZIP=NO ;;
@@ -394,6 +396,7 @@ cat << END
--with-http_random_index_module enable ngx_http_random_index_module
--with-http_secure_link_module enable ngx_http_secure_link_module
--with-http_degradation_module enable ngx_http_degradation_module
+ --with-http_slice_module enable ngx_http_slice_module
--with-http_stub_status_module enable ngx_http_stub_status_module
--without-http_charset_module disable ngx_http_charset_module
diff -r f44de0d12143 -r 29f35e60840b auto/sources
--- a/auto/sources Mon Dec 07 16:30:47 2015 +0300
+++ b/auto/sources Mon Dec 07 16:30:48 2015 +0300
@@ -360,6 +360,10 @@ HTTP_USERID_FILTER_MODULE=ngx_http_useri
HTTP_USERID_SRCS=src/http/modules/ngx_http_userid_filter_module.c
+HTTP_SLICE_FILTER_MODULE=ngx_http_slice_filter_module
+HTTP_SLICE_SRCS=src/http/modules/ngx_http_slice_filter_module.c
+
+
HTTP_REALIP_MODULE=ngx_http_realip_module
HTTP_REALIP_SRCS=src/http/modules/ngx_http_realip_module.c
diff -r f44de0d12143 -r 29f35e60840b src/http/modules/ngx_http_range_filter_module.c
--- a/src/http/modules/ngx_http_range_filter_module.c Mon Dec 07 16:30:47 2015 +0300
+++ b/src/http/modules/ngx_http_range_filter_module.c Mon Dec 07 16:30:48 2015 +0300
@@ -154,7 +154,7 @@ ngx_http_range_header_filter(ngx_http_re
if (r->http_version < NGX_HTTP_VERSION_10
|| r->headers_out.status != NGX_HTTP_OK
- || r != r->main
+ || (r != r->main && !r->subrequest_ranges)
|| r->headers_out.content_length_n == -1
|| !r->allow_ranges)
{
@@ -222,6 +222,8 @@ parse:
return NGX_ERROR;
}
+ ctx->offset = r->headers_out.content_offset;
+
if (ngx_array_init(&ctx->ranges, r->pool, 1, sizeof(ngx_http_range_t))
!= NGX_OK)
{
@@ -273,10 +275,21 @@ static ngx_int_t
ngx_http_range_parse(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx,
ngx_uint_t ranges)
{
- u_char *p;
- off_t start, end, size, content_length, cutoff, cutlim;
- ngx_uint_t suffix;
- ngx_http_range_t *range;
+ u_char *p;
+ off_t start, end, size, content_length, cutoff,
+ cutlim;
+ ngx_uint_t suffix;
+ ngx_http_range_t *range;
+ ngx_http_range_filter_ctx_t *mctx;
+
+ if (r != r->main) {
+ mctx = ngx_http_get_module_ctx(r->main,
+ ngx_http_range_body_filter_module);
+ if (mctx) {
+ ctx->ranges = mctx->ranges;
+ return NGX_OK;
+ }
+ }
p = r->headers_in.range->value.data + 6;
size = 0;
@@ -395,6 +408,10 @@ ngx_http_range_singlepart_header(ngx_htt
ngx_table_elt_t *content_range;
ngx_http_range_t *range;
+ if (r != r->main) {
+ return ngx_http_next_header_filter(r);
+ }
+
content_range = ngx_list_push(&r->headers_out.headers);
if (content_range == NULL) {
return NGX_ERROR;
@@ -422,6 +439,7 @@ ngx_http_range_singlepart_header(ngx_htt
- content_range->value.data;
r->headers_out.content_length_n = range->end - range->start;
+ r->headers_out.content_offset = range->start;
if (r->headers_out.content_length) {
r->headers_out.content_length->hash = 0;
diff -r f44de0d12143 -r 29f35e60840b src/http/modules/ngx_http_slice_filter_module.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/http/modules/ngx_http_slice_filter_module.c Mon Dec 07 16:30:48 2015 +0300
@@ -0,0 +1,521 @@
+
+/*
+ * Copyright (C) Roman Arutyunyan
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_http.h>
+
+
+typedef struct {
+ size_t size;
+} ngx_http_slice_loc_conf_t;
+
+
+typedef struct {
+ off_t start;
+ off_t end;
+ ngx_str_t range;
+ ngx_str_t etag;
+ ngx_uint_t last; /* unsigned last:1; */
+} ngx_http_slice_ctx_t;
+
+
+typedef struct {
+ off_t start;
+ off_t end;
+ off_t complete_length;
+} ngx_http_slice_content_range_t;
+
+
+static ngx_int_t ngx_http_slice_header_filter(ngx_http_request_t *r);
+static ngx_int_t ngx_http_slice_body_filter(ngx_http_request_t *r,
+ ngx_chain_t *in);
+static ngx_int_t ngx_http_slice_parse_content_range(ngx_http_request_t *r,
+ ngx_http_slice_content_range_t *cr);
+static ngx_int_t ngx_http_slice_range_variable(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data);
+static off_t ngx_http_slice_get_start(ngx_http_request_t *r);
+static void *ngx_http_slice_create_loc_conf(ngx_conf_t *cf);
+static char *ngx_http_slice_merge_loc_conf(ngx_conf_t *cf, void *parent,
+ void *child);
+static ngx_int_t ngx_http_slice_add_variables(ngx_conf_t *cf);
+static ngx_int_t ngx_http_slice_init(ngx_conf_t *cf);
+
+
+static ngx_command_t ngx_http_slice_filter_commands[] = {
+
+ { ngx_string("slice"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_size_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_slice_loc_conf_t, size),
+ NULL },
+
+ ngx_null_command
+};
+
+
+static ngx_http_module_t ngx_http_slice_filter_module_ctx = {
+ ngx_http_slice_add_variables, /* preconfiguration */
+ ngx_http_slice_init, /* postconfiguration */
+
+ NULL, /* create main configuration */
+ NULL, /* init main configuration */
+
+ NULL, /* create server configuration */
+ NULL, /* merge server configuration */
+
+ ngx_http_slice_create_loc_conf, /* create location configuration */
+ ngx_http_slice_merge_loc_conf /* merge location configuration */
+};
+
+
+ngx_module_t ngx_http_slice_filter_module = {
+ NGX_MODULE_V1,
+ &ngx_http_slice_filter_module_ctx, /* module context */
+ ngx_http_slice_filter_commands, /* module directives */
+ NGX_HTTP_MODULE, /* module type */
+ NULL, /* init master */
+ NULL, /* init module */
+ NULL, /* init process */
+ NULL, /* init thread */
+ NULL, /* exit thread */
+ NULL, /* exit process */
+ NULL, /* exit master */
+ NGX_MODULE_V1_PADDING
+};
+
+
+static ngx_str_t ngx_http_slice_range_name = ngx_string("slice_range");
+
+static ngx_http_output_header_filter_pt ngx_http_next_header_filter;
+static ngx_http_output_body_filter_pt ngx_http_next_body_filter;
+
+
+static ngx_int_t
+ngx_http_slice_header_filter(ngx_http_request_t *r)
+{
+ off_t end;
+ ngx_int_t rc;
+ ngx_table_elt_t *h;
+ ngx_http_slice_ctx_t *ctx;
+ ngx_http_slice_loc_conf_t *slcf;
+ ngx_http_slice_content_range_t cr;
+
+ ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);
+ if (ctx == NULL) {
+ return ngx_http_next_header_filter(r);
+ }
+
+ if (r->headers_out.status != NGX_HTTP_PARTIAL_CONTENT) {
+ if (r == r->main) {
+ ngx_http_set_ctx(r, NULL, ngx_http_slice_filter_module);
+ return ngx_http_next_header_filter(r);
+ }
+
+ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+ "unexpected status code %ui in slice response",
+ r->headers_out.status);
+ return NGX_ERROR;
+ }
+
+ h = r->headers_out.etag;
+
+ if (ctx->etag.len) {
+ if (h == NULL
+ || h->value.len != ctx->etag.len
+ || ngx_strncmp(h->value.data, ctx->etag.data, ctx->etag.len)
+ != 0)
+ {
+ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+ "etag mismatch in slice response");
+ return NGX_ERROR;
+ }
+ }
+
+ if (h) {
+ ctx->etag = h->value;
+ }
+
More information about the nginx-devel
mailing list