[PATCH] Slice filter: proxy_cache_background_update support (ticket #1348)

J Carter jordanc.carter at outlook.com
Sat Jun 8 23:03:09 UTC 2024


# HG changeset patch
# User J Carter <jordanc.carter at outlook.com>
# Date 1717886685 -3600
#      Sat Jun 08 23:44:45 2024 +0100
# Node ID 1b8a60f7640be4a900ac77d8022b7d8cc6944186
# Parent  02e9411009b987f408214ab4a8b6b6093f843bcd
Slice filter: proxy_cache_background_update support (ticket #1348).

Previously, subrequests of a slice subrequest would have an empty
$slice_range variable value. This prevented
proxy_cache_background_update and friends from successfully
fetching and populating correctly.

This occurred for two reasons:
- Firstly, a single context was reused for all slice subrequests,
where each $slice_range value was overwritten by subsequent slice
subrequests.
- Secondly, subrequests not initiated by slice filter were unable to
access $slice_range in a parent subrequest.

Each slice subrequests now retains $slice_range and subrequests of
slice subrequests now utilize the parent slice subrequest's
$slice_range if available.

diff --git a/src/http/modules/ngx_http_slice_filter_module.c b/src/http/modules/ngx_http_slice_filter_module.c
--- a/src/http/modules/ngx_http_slice_filter_module.c
+++ b/src/http/modules/ngx_http_slice_filter_module.c
@@ -18,11 +18,16 @@ typedef struct {
 typedef struct {
     off_t                start;
     off_t                end;
-    ngx_str_t            range;
     ngx_str_t            etag;
     unsigned             last:1;
     unsigned             active:1;
     ngx_http_request_t  *sr;
+} ngx_http_slice_shctx_t;
+
+
+typedef struct {
+    ngx_str_t range;
+    ngx_http_slice_shctx_t *sh;
 } ngx_http_slice_ctx_t;
 
 
@@ -105,6 +110,7 @@ ngx_http_slice_header_filter(ngx_http_re
     ngx_int_t                        rc;
     ngx_table_elt_t                 *h;
     ngx_http_slice_ctx_t            *ctx;
+    ngx_http_slice_shctx_t          *sh;
     ngx_http_slice_loc_conf_t       *slcf;
     ngx_http_slice_content_range_t   cr;
 
@@ -113,6 +119,8 @@ ngx_http_slice_header_filter(ngx_http_re
         return ngx_http_next_header_filter(r);
     }
 
+    sh = ctx->sh;
+
     if (r->headers_out.status != NGX_HTTP_PARTIAL_CONTENT) {
         if (r == r->main) {
             ngx_http_set_ctx(r, NULL, ngx_http_slice_filter_module);
@@ -127,10 +135,10 @@ ngx_http_slice_header_filter(ngx_http_re
 
     h = r->headers_out.etag;
 
-    if (ctx->etag.len) {
+    if (sh->etag.len) {
         if (h == NULL
-            || h->value.len != ctx->etag.len
-            || ngx_strncmp(h->value.data, ctx->etag.data, ctx->etag.len)
+            || h->value.len != sh->etag.len
+            || ngx_strncmp(h->value.data, sh->etag.data, sh->etag.len)
                != 0)
         {
             ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
@@ -140,7 +148,7 @@ ngx_http_slice_header_filter(ngx_http_re
     }
 
     if (h) {
-        ctx->etag = h->value;
+        sh->etag = h->value;
     }
 
     if (ngx_http_slice_parse_content_range(r, &cr) != NGX_OK) {
@@ -163,15 +171,15 @@ ngx_http_slice_header_filter(ngx_http_re
 
     end = ngx_min(cr.start + (off_t) slcf->size, cr.complete_length);
 
-    if (cr.start != ctx->start || cr.end != end) {
+    if (cr.start != sh->start || cr.end != end) {
         ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                       "unexpected range in slice response: %O-%O",
                       cr.start, cr.end);
         return NGX_ERROR;
     }
 
-    ctx->start = end;
-    ctx->active = 1;
+    sh->start = end;
+    sh->active = 1;
 
     r->headers_out.status = NGX_HTTP_OK;
     r->headers_out.status_line.len = 0;
@@ -198,16 +206,16 @@ ngx_http_slice_header_filter(ngx_http_re
     r->preserve_body = 1;
 
     if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) {
-        if (ctx->start + (off_t) slcf->size <= r->headers_out.content_offset) {
-            ctx->start = slcf->size
+        if (sh->start + (off_t) slcf->size <= r->headers_out.content_offset) {
+            sh->start = slcf->size
                          * (r->headers_out.content_offset / slcf->size);
         }
 
-        ctx->end = r->headers_out.content_offset
+        sh->end = r->headers_out.content_offset
                    + r->headers_out.content_length_n;
 
     } else {
-        ctx->end = cr.complete_length;
+        sh->end = cr.complete_length;
     }
 
     return rc;
@@ -217,9 +225,11 @@ ngx_http_slice_header_filter(ngx_http_re
 static ngx_int_t
 ngx_http_slice_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
 {
+    u_char                     *p;
     ngx_int_t                   rc;
     ngx_chain_t                *cl;
-    ngx_http_slice_ctx_t       *ctx;
+    ngx_http_slice_ctx_t       *ctx, *sr_ctx;
+    ngx_http_slice_shctx_t     *sh;
     ngx_http_slice_loc_conf_t  *slcf;
 
     ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);
@@ -228,32 +238,34 @@ ngx_http_slice_body_filter(ngx_http_requ
         return ngx_http_next_body_filter(r, in);
     }
 
+    sh = ctx->sh;
+
     for (cl = in; cl; cl = cl->next) {
         if (cl->buf->last_buf) {
             cl->buf->last_buf = 0;
             cl->buf->last_in_chain = 1;
             cl->buf->sync = 1;
-            ctx->last = 1;
+            sh->last = 1;
         }
     }
 
     rc = ngx_http_next_body_filter(r, in);
 
-    if (rc == NGX_ERROR || !ctx->last) {
+    if (rc == NGX_ERROR || !sh->last) {
         return rc;
     }
 
-    if (ctx->sr && !ctx->sr->done) {
+    if (sh->sr && !sh->sr->done) {
         return rc;
     }
 
-    if (!ctx->active) {
+    if (!sh->active) {
         ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                       "missing slice response");
         return NGX_ERROR;
     }
 
-    if (ctx->start >= ctx->end) {
+    if (sh->start >= sh->end) {
         ngx_http_set_ctx(r, NULL, ngx_http_slice_filter_module);
         ngx_http_send_special(r, NGX_HTTP_LAST);
         return rc;
@@ -263,25 +275,36 @@ ngx_http_slice_body_filter(ngx_http_requ
         return rc;
     }
 
-    if (ngx_http_subrequest(r, &r->uri, &r->args, &ctx->sr, NULL,
+    if (ngx_http_subrequest(r, &r->uri, &r->args, &sh->sr, NULL,
                             NGX_HTTP_SUBREQUEST_CLONE)
         != NGX_OK)
     {
         return NGX_ERROR;
     }
 
-    ngx_http_set_ctx(ctx->sr, ctx, ngx_http_slice_filter_module);
+    sr_ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_slice_ctx_t));
+    if (sr_ctx == NULL) {
+        return NGX_ERROR;
+    }
+
+    ngx_http_set_ctx(sh->sr, sr_ctx, ngx_http_slice_filter_module);
 
     slcf = ngx_http_get_module_loc_conf(r, ngx_http_slice_filter_module);
 
-    ctx->range.len = ngx_sprintf(ctx->range.data, "bytes=%O-%O", ctx->start,
-                                 ctx->start + (off_t) slcf->size - 1)
-                     - ctx->range.data;
+    p = ngx_pnalloc(r->pool, sizeof("bytes=-") - 1 + 2 * NGX_OFF_T_LEN);
+    if (p == NULL) {
+        return NGX_ERROR;
+    }
 
-    ctx->active = 0;
+    sr_ctx->range.data = p;
+    sr_ctx->range.len = ngx_sprintf(p, "bytes=%O-%O", sh->start,
+                                    sh->start + (off_t) slcf->size - 1) - p;
+
+    sh->active = 0;
+    sr_ctx->sh = ctx->sh;
 
     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                   "http slice subrequest: \"%V\"", &ctx->range);
+                   "http slice subrequest: \"%V\"", &sr_ctx->range);
 
     return rc;
 }
@@ -397,12 +420,25 @@ ngx_http_slice_range_variable(ngx_http_r
 {
     u_char                     *p;
     ngx_http_slice_ctx_t       *ctx;
+    ngx_http_slice_shctx_t     *sh;
     ngx_http_slice_loc_conf_t  *slcf;
 
     ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);
 
     if (ctx == NULL) {
-        if (r != r->main || r->headers_out.status) {
+        if (r != r->main) {
+            if (r->headers_out.status) {
+                v->not_found = 1;
+                return NGX_OK;
+            }
+
+            /* cache_background_update handling */
+            ctx = ngx_http_get_module_ctx(r->parent,
+                                          ngx_http_slice_filter_module);
+            if (ctx) {
+                goto found;
+            }
+
             v->not_found = 1;
             return NGX_OK;
         }
@@ -421,19 +457,27 @@ ngx_http_slice_range_variable(ngx_http_r
 
         ngx_http_set_ctx(r, ctx, ngx_http_slice_filter_module);
 
+        sh = ngx_pcalloc(r->pool, sizeof(ngx_http_slice_shctx_t));
+        if (sh == NULL) {
+            return NGX_ERROR;
+        }
+
+        ctx->sh = sh;
+
         p = ngx_pnalloc(r->pool, sizeof("bytes=-") - 1 + 2 * NGX_OFF_T_LEN);
         if (p == NULL) {
             return NGX_ERROR;
         }
 
-        ctx->start = slcf->size * (ngx_http_slice_get_start(r) / slcf->size);
+        sh->start = slcf->size * (ngx_http_slice_get_start(r) / slcf->size);
 
         ctx->range.data = p;
-        ctx->range.len = ngx_sprintf(p, "bytes=%O-%O", ctx->start,
-                                     ctx->start + (off_t) slcf->size - 1)
-                         - p;
+        ctx->range.len = ngx_sprintf(p, "bytes=%O-%O", sh->start,
+                                     sh->start + (off_t) slcf->size - 1) - p;
     }
 
+found:
+
     v->data = ctx->range.data;
     v->valid = 1;
     v->not_found = 0;


More information about the nginx-devel mailing list