[patch] Range filter: support multiple ranges.
胡聪 (hucc)
hucong.c at foxmail.com
Fri Oct 27 10:45:32 UTC 2017
Hi,
Please ignore this patch, I will update later. I am sorry！
------------------ Original ------------------
From: "胡聪 (hucc)";<hucong.c at foxmail.com>;
Date: Oct 27, 2017
To: "nginx-devel"<nginx-devel at nginx.org>;
Subject: [patch] Range filter: support multiple ranges.
# HG changeset patch
# User hucongcong <hucong.c at foxmail.com>
# Date 1509038506 -28800
# Fri Oct 27 01:21:46 2017 +0800
# Node ID 798418b3f9c21811debf6cbeaea3eb554c886905
# Parent a429a3b21a3950d69781e109111831e7c5f33566
Range filter: support multiple ranges.
Nginx will return 416 when the ranges (first-byte-pos) are not listed in
ascending order or when one byte-range-spec contains another. And besides,
nginx will coalesce any of the ranges that overlap, or that are separated
by a gap that is smaller than the NGX_HTTP_RANGE_MULTIPART_GAP macro.
diff -r a429a3b21a39 -r 798418b3f9c2 src/http/modules/ngx_http_range_filter_module.c
--- a/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 00:38:56 2017 +0800
+++ b/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 01:21:46 2017 +0800
@@ -10,6 +10,9 @@
#include <ngx_http.h>
+#define NGX_HTTP_RANGE_MULTIPART_GAP 80
+
+
/*
* the single part format:
*
@@ -55,6 +58,7 @@ typedef struct {
typedef struct {
off_t offset;
ngx_str_t boundary_header;
+ ngx_uint_t index; /* start with 1 */
ngx_array_t ranges;
} ngx_http_range_filter_ctx_t;
@@ -66,12 +70,14 @@ static ngx_int_t ngx_http_range_singlepa
static ngx_int_t ngx_http_range_multipart_header(ngx_http_request_t *r,
ngx_http_range_filter_ctx_t *ctx);
static ngx_int_t ngx_http_range_not_satisfiable(ngx_http_request_t *r);
-static ngx_int_t ngx_http_range_test_overlapped(ngx_http_request_t *r,
- ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in);
static ngx_int_t ngx_http_range_singlepart_body(ngx_http_request_t *r,
ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in);
static ngx_int_t ngx_http_range_multipart_body(ngx_http_request_t *r,
ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in);
+static ngx_int_t ngx_http_range_link_boundary_header(ngx_http_request_t *r,
+ ngx_http_range_filter_ctx_t *ctx, ngx_chain_t ***lll);
+static ngx_int_t ngx_http_range_link_last_boundary(ngx_http_request_t *r,
+ ngx_http_range_filter_ctx_t *ctx, ngx_chain_t **ll);
static ngx_int_t ngx_http_range_header_filter_init(ngx_conf_t *cf);
static ngx_int_t ngx_http_range_body_filter_init(ngx_conf_t *cf);
@@ -270,9 +276,9 @@ ngx_http_range_parse(ngx_http_request_t
ngx_uint_t ranges)
{
u_char *p;
- off_t start, end, size, content_length, cutoff,
- cutlim;
- ngx_uint_t suffix;
+ off_t start, end, size, content_length,
+ suffix_length, cutoff, cutlim;
+ ngx_uint_t i, suffix;
ngx_http_range_t *range;
ngx_http_range_filter_ctx_t *mctx;
@@ -280,6 +286,7 @@ ngx_http_range_parse(ngx_http_request_t
mctx = ngx_http_get_module_ctx(r->main,
ngx_http_range_body_filter_module);
if (mctx) {
+ ctx->boundary_header = mctx->boundary_header;
ctx->ranges = mctx->ranges;
return NGX_OK;
}
@@ -293,6 +300,8 @@ ngx_http_range_parse(ngx_http_request_t
p = r->headers_in.range->value.data + 6;
size = 0;
+ range = NULL;
+ suffix_length = 0;
content_length = r->headers_out.content_length_n;
cutoff = NGX_MAX_OFF_T_VALUE / 10;
@@ -374,6 +383,94 @@ ngx_http_range_parse(ngx_http_request_t
return NGX_DECLINED;
}
+ if (suffix) {
+
+ if (end - start > suffix_length) {
+ suffix_length = end - start;
+ }
+
+ } else {
+
+ if (ctx->ranges.nelts) {
+
+ if (start < range->start) {
+ ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
+ "descending order ranges %O-%O, %O-%O",
+ range->start, range->end - 1,
+ start, end - 1);
+
+ return NGX_HTTP_RANGE_NOT_SATISFIABLE;
+ }
+
+ if (end <= range->end) {
+ ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
+ "inclusive ranges %O-%O, %O-%O",
+ range->start, range->end - 1,
+ start, end - 1);
+
+ return NGX_HTTP_RANGE_NOT_SATISFIABLE;
+ }
+
+ if (start < range->end + NGX_HTTP_RANGE_MULTIPART_GAP) {
+ size -= range->end - range->start;
+ start = range->start;
+ ctx->ranges.nelts--;
+ }
+ }
+
+ range = ngx_array_push(&ctx->ranges);
+ if (range == NULL) {
+ return NGX_ERROR;
+ }
+
+ range->start = start;
+ range->end = end;
+
+ if (size > NGX_MAX_OFF_T_VALUE - (end - start)) {
+ return NGX_HTTP_RANGE_NOT_SATISFIABLE;
+ }
+
+ size += end - start;
+ }
+
+ } else if (start == 0) {
+ return NGX_DECLINED;
+ }
+
+ if (*p++ != ',') {
+ break;
+ }
+ }
+
+ if (suffix_length) {
+ start = content_length - suffix_length;
+ end = content_length;
+
+ i = 0;
+ range = ctx->ranges.elts;
+
+ while (ctx->ranges.nelts) {
+
+ i = ctx->ranges.nelts - 1;
+ if (start >= range[i].start) {
+ break;
+ }
+
+ size -= range[i].end - range[i].start;
+ ctx->ranges.nelts--;
+ }
+
+ if (ctx->ranges.nelts
+ && start < range[i].end + NGX_HTTP_RANGE_MULTIPART_GAP)
+ {
+ size -= range[i].end - range[i].start;
+ start = range[i].start;
+ ctx->ranges.nelts--;
+ i--;
+ }
+
+ if (ctx->ranges.nelts == 0 || end > range[i].end) {
+
range = ngx_array_push(&ctx->ranges);
if (range == NULL) {
return NGX_ERROR;
@@ -387,13 +484,6 @@ ngx_http_range_parse(ngx_http_request_t
}
size += end - start;
-
- } else if (start == 0) {
- return NGX_DECLINED;
- }
-
- if (*p++ != ',') {
- break;
}
}
@@ -470,6 +560,10 @@ ngx_http_range_multipart_header(ngx_http
ngx_http_range_t *range;
ngx_atomic_uint_t boundary;
+ if (r != r->main) {
+ return ngx_http_next_header_filter(r);
+ }
+
size = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN
+ sizeof(CRLF "Content-Type: ") - 1
+ r->headers_out.content_type.len
@@ -571,10 +665,11 @@ ngx_http_range_multipart_header(ngx_http
- range[i].content_range.data;
len += ctx->boundary_header.len + range[i].content_range.len
- + (range[i].end - range[i].start);
+ + (range[i].end - range[i].start);
}
r->headers_out.content_length_n = len;
+ r->headers_out.content_offset = range[0].start;
if (r->headers_out.content_length) {
r->headers_out.content_length->hash = 0;
@@ -640,63 +735,15 @@ ngx_http_range_body_filter(ngx_http_requ
return ngx_http_range_singlepart_body(r, ctx, in);
}
- /*
- * multipart ranges are supported only if whole body is in a single buffer
- */
-
if (ngx_buf_special(in->buf)) {
return ngx_http_next_body_filter(r, in);
}
- if (ngx_http_range_test_overlapped(r, ctx, in) != NGX_OK) {
- return NGX_ERROR;
- }
-
return ngx_http_range_multipart_body(r, ctx, in);
}
static ngx_int_t
-ngx_http_range_test_overlapped(ngx_http_request_t *r,
- ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in)
-{
- off_t start, last;
- ngx_buf_t *buf;
- ngx_uint_t i;
- ngx_http_range_t *range;
-
- if (ctx->offset) {
- goto overlapped;
- }
-
- buf = in->buf;
-
- if (!buf->last_buf) {
- start = ctx->offset;
- last = ctx->offset + ngx_buf_size(buf);
-
- range = ctx->ranges.elts;
- for (i = 0; i < ctx->ranges.nelts; i++) {
- if (start > range[i].start || last < range[i].end) {
- goto overlapped;
- }
- }
- }
-
- ctx->offset = ngx_buf_size(buf);
-
- return NGX_OK;
-
-overlapped:
-
- ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
- "range in overlapped buffers");
-
- return NGX_ERROR;
-}
-
-
-static ngx_int_t
ngx_http_range_singlepart_body(ngx_http_request_t *r,
ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in)
{
@@ -787,96 +834,225 @@ static ngx_int_t
ngx_http_range_multipart_body(ngx_http_request_t *r,
ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in)
{
- ngx_buf_t *b, *buf;
- ngx_uint_t i;
- ngx_chain_t *out, *hcl, *rcl, *dcl, **ll;
- ngx_http_range_t *range;
+ off_t start, last, back;
+ ngx_buf_t *buf, *b;
+ ngx_uint_t i, finished;
+ ngx_chain_t *out, *cl, *ncl, **ll;
+ ngx_http_range_t *range, *tail;
- ll = &out;
- buf = in->buf;
range = ctx->ranges.elts;
- for (i = 0; i < ctx->ranges.nelts; i++) {
+ if (!ctx->index) {
+ for (i = 0; i < ctx->ranges.nelts; i++) {
+ if (ctx->offset < range[i].end) {
+ ctx->index = i + 1;
+ break;
+ }
+ }
+ }
+
+ tail = range + ctx->ranges.nelts - 1;
+ range += ctx->index - 1;
+
+ out = NULL;
+ ll = &out;
+ finished = 0;
- /*
- * The boundary header of the range:
- * CRLF
- * "--0123456789" CRLF
- * "Content-Type: image/jpeg" CRLF
- * "Content-Range: bytes "
- */
+ for (cl = in; cl; cl = cl->next) {
+
+ buf = cl->buf;
+
+ start = ctx->offset;
+ last = ctx->offset + ngx_buf_size(buf);
- b = ngx_calloc_buf(r->pool);
- if (b == NULL) {
- return NGX_ERROR;
+ ctx->offset = last;
+
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "http range multipart body buf: %O-%O", start, last);
+
+ if (ngx_buf_special(buf)) {
+ *ll = cl;
+ ll = &cl->next;
+ continue;
}
- b->memory = 1;
- b->pos = ctx->boundary_header.data;
- b->last = ctx->boundary_header.data + ctx->boundary_header.len;
+ if (range->end <= start || range->start >= last) {
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "http range multipart body skip");
- hcl = ngx_alloc_chain_link(r->pool);
- if (hcl == NULL) {
- return NGX_ERROR;
+ if (buf->in_file) {
+ buf->file_pos = buf->file_last;
+ }
+
+ buf->pos = buf->last;
+ buf->sync = 1;
+
+ continue;
}
- hcl->buf = b;
+ if (range->start >= start) {
+ if (ngx_http_range_link_boundary_header(r, ctx, &ll) != NGX_OK) {
+ return NGX_ERROR;
+ }
- /* "SSSS-EEEE/TTTT" CRLF CRLF */
+ if (buf->in_file) {
+ buf->file_pos += range->start - start;
+ }
- b = ngx_calloc_buf(r->pool);
- if (b == NULL) {
- return NGX_ERROR;
+ if (ngx_buf_in_memory(buf)) {
+ buf->pos += (size_t) (range->start - start);
+ }
}
- b->temporary = 1;
- b->pos = range[i].content_range.data;
- b->last = range[i].content_range.data + range[i].content_range.len;
+ if (range->end <= last) {
+
+ if (range < tail && range[1].start < last) {
+
+ b = ngx_alloc_buf(r->pool);
+ if (b == NULL) {
+ return NGX_ERROR;
+ }
+
+ ncl = ngx_alloc_chain_link(r->pool);
+ if (ncl == NULL) {
+ return NGX_ERROR;
+ }
- rcl = ngx_alloc_chain_link(r->pool);
- if (rcl == NULL) {
- return NGX_ERROR;
- }
+ ncl->buf = b;
+ ncl->next = cl->next;
+ cl->next = ncl;
+
+ ngx_memcpy(b, buf, sizeof(ngx_buf_t));
+ buf->last_in_chain = 0;
+ buf->last_buf = 0;
+
+ back = last - range->end;
+ ctx->offset -= back;
+
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "http range multipart body reuse buf: %O-%O",
+ ctx->offset, ctx->offset + back);
- rcl->buf = b;
+ if (b->in_file) {
+ b->file_pos = b->file_last - back;
+ }
+ if (ngx_buf_in_memory(b)) {
+ b->pos = b->last - back;
+ }
+ }
+
+ if (buf->in_file) {
+ buf->file_last -= last - range->end;
+ }
- /* the range data */
+ if (ngx_buf_in_memory(buf)) {
+ buf->last -= (size_t) (last - range->end);
+ }
- b = ngx_calloc_buf(r->pool);
- if (b == NULL) {
- return NGX_ERROR;
+ if (range == tail) {
+ buf->last_buf = (r == r->main) ? 1 : 0;
+ buf->last_in_chain = 1;
+ *ll = cl;
+ ll = &cl->next;
+
+ finished = 1;
+ break;
+ }
+
+ range++;
+ ctx->index++;
}
- b->in_file = buf->in_file;
- b->temporary = buf->temporary;
- b->memory = buf->memory;
- b->mmap = buf->mmap;
- b->file = buf->file;
+ *ll = cl;
+ ll = &cl->next;
+ }
+
+ if (out == NULL) {
+ return NGX_OK;
+ }
+
+ *ll = NULL;
+
+ if (finished
+ && ngx_http_range_link_last_boundary(r, ctx, ll) != NGX_OK)
+ {
+ return NGX_ERROR;
+ }
+
+ return ngx_http_next_body_filter(r, out);
+}
+
- if (buf->in_file) {
- b->file_pos = buf->file_pos + range[i].start;
- b->file_last = buf->file_pos + range[i].end;
- }
+static ngx_int_t
+ngx_http_range_link_boundary_header(ngx_http_request_t *r,
+ ngx_http_range_filter_ctx_t *ctx, ngx_chain_t ***lll)
+{
+ ngx_buf_t *b;
+ ngx_chain_t *hcl, *rcl;
+ ngx_http_range_t *range;
+
+ /*
+ * The boundary header of the range:
+ * CRLF
+ * "--0123456789" CRLF
+ * "Content-Type: image/jpeg" CRLF
+ * "Content-Range: bytes "
+ */
+
+ b = ngx_calloc_buf(r->pool);
+ if (b == NULL) {
+ return NGX_ERROR;
+ }
- if (ngx_buf_in_memory(buf)) {
- b->pos = buf->pos + (size_t) range[i].start;
- b->last = buf->pos + (size_t) range[i].end;
- }
+ b->memory = 1;
+ b->pos = ctx->boundary_header.data;
+ b->last = ctx->boundary_header.data + ctx->boundary_header.len;
+
+ hcl = ngx_alloc_chain_link(r->pool);
+ if (hcl == NULL) {
+ return NGX_ERROR;
+ }
+
+ hcl->buf = b;
+
+
+ /* "SSSS-EEEE/TTTT" CRLF CRLF */
+
+ b = ngx_calloc_buf(r->pool);
+ if (b == NULL) {
+ return NGX_ERROR;
+ }
- dcl = ngx_alloc_chain_link(r->pool);
- if (dcl == NULL) {
- return NGX_ERROR;
- }
+ range = ctx->ranges.elts;
+ b->temporary = 1;
+ b->pos = range[ctx->index - 1].content_range.data;
+ b->last = range[ctx->index - 1].content_range.data
+ + range[ctx->index - 1].content_range.len;
+
+ rcl = ngx_alloc_chain_link(r->pool);
+ if (rcl == NULL) {
+ return NGX_ERROR;
+ }
+
+ rcl->buf = b;
- dcl->buf = b;
+ **lll = hcl;
+ hcl->next = rcl;
+ *lll = &rcl->next;
+
+ return NGX_OK;
+}
- *ll = hcl;
- hcl->next = rcl;
- rcl->next = dcl;
- ll = &dcl->next;
- }
+
+static ngx_int_t
+ngx_http_range_link_last_boundary(ngx_http_request_t *r,
+ ngx_http_range_filter_ctx_t *ctx, ngx_chain_t **ll)
+{
+ ngx_buf_t *b;
+ ngx_chain_t *hcl;
/* the last boundary CRLF "--0123456789--" CRLF */
@@ -886,7 +1062,8 @@ ngx_http_range_multipart_body(ngx_http_r
}
b->temporary = 1;
- b->last_buf = 1;
+ b->last_in_chain = 1;
+ b->last_buf = (r == r->main) ? 1 : 0;
b->pos = ngx_pnalloc(r->pool, sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN
+ sizeof("--" CRLF) - 1);
@@ -909,7 +1086,7 @@ ngx_http_range_multipart_body(ngx_http_r
*ll = hcl;
- return ngx_http_next_body_filter(r, out);
+ return NGX_OK;
}
_______________________________________________
nginx-devel mailing list
nginx-devel at nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel
More information about the nginx-devel
mailing list