Throttle requests with limit_req rate based on header from response to auth subrequest

jarstewa nginx-forum at forum.nginx.org
Wed Aug 29 23:14:01 UTC 2018


I'm hoping to use the limit_req directive with different rates based on a
header that is returned from the auth subrequest.  I got some ideas from
https://www.ruby-forum.com/topic/4418040 but am running into a few problems.
 Here is my configuration: 

> user                     nginx;
> worker_processes         auto;
> error_log                /var/log/nginx/error.log warn;
> pid                      /var/run/nginx.pid;
> events {
>   worker_connections     10000;
> }
> 
> worker_rlimit_nofile 10000;
> 
> http {
>   log_subrequest on;
> 
>   log_format  main  escape=json  '{ "timestamp": "$time_local", "client":
"$remote_addr",'
>                               ' "method": "$request_method", "uri":
"$uri",'
>                               ' "request_length": $request_length,'
>                               ' "status": $status, "bytes_sent":
$bytes_sent,'
>                               ' "upstream_status": "$upstream_status",'
>                               ' "request_id": "$request_id",'
>                               ' "request_uri": "$request_uri",'
>                               ' "tier": "$tier",'
>                               ' "upstream_http_tier":
"$upstream_http_tier",'
>                               ' "2X_key": "$2X_key",'
>                               ' "3X_key": "$3X_key",'
>                               ' "2X_key_from_upstream":
"$2X_key_from_upstream",'
>                               ' "3X_key_from_upstream":
"$3X_key_from_upstream",'
>                               ' "origin": "$http_origin"}' ;
> 
>   access_log  /var/log/nginx/access.log  main;
>   sendfile               on;
>   tcp_nopush             on;
>   tcp_nodelay            on;
>   keepalive_timeout      65;
>   types_hash_max_size    2048;
>   include                /etc/nginx/mime.types;
>   default_type           application/octet-stream;
>   proxy_buffering        on;
>   proxy_buffers          8 64k;
>   proxy_cache_path       /dev/shm/nginx/auth use_temp_path=off levels=1:2
keys_zone=auth_cache:1024m inactive=30m max_size=1g;
>   proxy_cache_path       /dev/shm/nginx/manifests use_temp_path=off
levels=1:2 keys_zone=manifest_cache:100m inactive=30s max_size=10g;
>   proxy_cache_methods    GET HEAD;
>   proxy_cache_lock       on;
>   proxy_cache_use_stale  updating;
>   proxy_bind             0.0.0.0;
>   proxy_ignore_headers   Expires;
>   proxy_pass_header Server;
> 
> 
>   map $request_uri $endpoint_id {
>     default "unknown";
>     ~^/out/v\d+/(?P<endpoint.+?)/.+$ $endpoint;
>   }
> 
>   # Mappings based on the tier header from the /auth request
>   map $tier $2X_key {~02x $endpoint_id; default "";}
>   map $tier $3X_key {~03x $endpoint_id; default "";}
>   map $upstream_http_tier $2X_key_from_upstream {~02x $endpoint_id;
default "";}
>   map $upstream_http_tier $3X_key_from_upstream {~03x $endpoint_id;
default "";}
> 
>   # Throttle zones based on the results of the above mapping
>   limit_req_zone $2X_key zone=2x_zone:20m rate=10r/s;
>   limit_req_zone $3X_key zone=3x_zone:20m rate=100r/s;
>   limit_req_zone $2X_key_from_upstream zone=2x_zone_from_upstream:20m
rate=10r/s;
>   limit_req_zone $3X_key_from_upstream zone=3x_zone_from_upstream:20m
rate=100r/s;
> 
> 
>   server {
>     listen               80 default_server;
>     listen               [::]:80 default_server;
>     server_name          default_backend;
>     server_tokens        off;
>     access_log           /var/log/nginx/access.log main;
> 
>     root /var/www/html;
> 
>     error_page 401                    /error_pages/401.html;
>     error_page 403                    /error_pages/403.html;
>     error_page 404                    /error_pages/404.html;
>     error_page 429                    /error_pages/429.html;
>     error_page 500 501 502 503 504    /error_pages/5xx.html;
> 
>     set $upstream_server http://my_server:80;
> 
>     proxy_http_version 1.1;
>     proxy_set_header   Connection "";
>     proxy_connect_timeout       10;
>     proxy_send_timeout          30;
>     proxy_read_timeout          30;
> 
>     proxy_cache_valid  404 412 1s;
> 
>     location ~* \.(m3u8|mpd|isml?/manifest)$ {
>       auth_request       /auth;
> 
>       # Capture the tier header from auth request
>       auth_request_set $tier $upstream_http_tier;
> 
>       # Throttling based on mappings from tier
>       limit_req zone=2x_zone burst=10 nodelay;
>       limit_req zone=3x_zone burst=10 nodelay;
>       limit_req zone=2x_zone_from_upstream burst=10 nodelay;
>       limit_req zone=3x_zone_from_upstream burst=10 nodelay;
>       limit_req_status 429;
> 
>       proxy_pass         $upstream_server;
>       proxy_cache        manifest_cache;
>       set $cache_key "${endpoint_id}";
>       proxy_cache_key    $cache_key;
>       proxy_cache_valid  200 301 302  2s;
> 
>       access_log         /var/log/nginx/access.log  main;
>     }
> 
>     location /auth {
>       internal;
> 
>       set $auth_type 1;
>       proxy_pass_request_body off;
>       proxy_pass         $upstream_server/auth?endpoint=$endpoint_id;
> 
>       proxy_cache        auth_cache;
>       set $auth_cache_key "${endpoint_id}";
>       proxy_cache_key    $auth_cache_key;
>       proxy_cache_valid  200 301 302  5m;
>       proxy_cache_valid  400 401 403 404  5m;
> 
>       access_log         /var/log/nginx/access.log  main;
>     }
>   }
> }
> 

I'm expecting the following line to capture the "tier" header that comes
back from the  auth subrequest (sometimes it will be "02x", and sometimes
"03x"): 

>  auth_request_set $tier $upstream_http_tier; 

Then, that value will be passed into several mappings that should either
return "" or a value, depending on which throttle zone should be applied. 
(There are variants using both $tier and $upstream_http_tier from attempts
at troubleshooting.) 
 
>  map $tier $2X_key {~02x $endpoint_id; default "";} 
>  map $tier $3X_key {~03x $endpoint_id; default "";} 
>  map $upstream_http_tier $2X_key_from_upstream {~02x $endpoint_id; default
"";} 
>  map $upstream_http_tier $3X_key_from_upstream {~03x $endpoint_id; default
"";} 

Finally, I expect only the limit_req directive with a non-empty key to be
applied:

>  limit_req_zone $2X_key zone=2x_zone:20m rate=10r/s; 
>  limit_req_zone $3X_key zone=3x_zone:20m rate=100r/s; 
>  limit_req_zone $2X_key_from_upstream zone=2x_zone_from_upstream:20m
rate=10r/s; 
>  limit_req_zone $3X_key_from_upstream zone=3x_zone_from_upstream:20m
rate=100r/s; 
 

However, it's look like at least two things are not behaving as I expect. 
1) The only throttling that I see is coming from 2x_zone, even when the
header comes back as "03x".

2018/08/29 22:37:31 [error] 12626#0: *1596735 limiting requests, excess:
10.010 by zone "2x_zone", client: 10.0.136.178, server: default_backend,
request: "GET /out/v1/04b1719535444a1d84389aeb0a1fb912/throttleCanary.m3u8
HTTP/1.1", host: "myserver"

2) Mapping based on the variable captured from auth_request_set don't appear
to be set as I expect on the main request.

Here's what I see in the access log for an auth request and a main request:


{
    "timestamp": "29/Aug/2018:22:37:31 +0000",
    "client": "10.0.136.178",
    "method": "GET",
    "uri": "/auth",
    "status": 200,
    "bytes_sent": 0,
    "upstream_status": "",
    "server_name": "default_backend",
    "request_id": "d6545f5758c2b3944438c80f2e964678",
    "request_uri":
"/out/v1/04b1719535444a1d84389aeb0a1fb912/throttleCanary.m3u8",
    "tier": "",
    "upstream_http_tier": "02x",
    "2X_key": "",
    "3X_key": "",
    "2X_key_from_upstream": "04b1719535444a1d84389aeb0a1fb912",
    "3X_key_from_upstream": "",
    "origin": ""
}
{
    "timestamp": "29/Aug/2018:22:37:31 +0000",
    "client": "10.0.136.178",
    "method": "GET",
    "uri": "/out/v1/04b1719535444a1d84389aeb0a1fb912/throttleCanary.m3u8",
    "status": 200,
    "bytes_sent": 652,
    "upstream_status": "",
    "server_name": "default_backend",
    "request_id": "d6545f5758c2b3944438c80f2e964678",
    "request_uri":
"/out/v1/04b1719535444a1d84389aeb0a1fb912/throttleCanary.m3u8",
    "tier": "02x",
    "upstream_http_tier": "",
    "2X_key": "",
    "3X_key": "",
    "2X_key_from_upstream": "04b1719535444a1d84389aeb0a1fb912",
    "3X_key_from_upstream": "",
    "origin": ""
}

Notice that in the main request, "tier" is "02x" as expected, but the
mapping based on $tier (2X_key) is empty while the mapping based on
$http_upstream_tier (2X_key_from_upstream) has a value.  Moreover, since
only 2X_key_from_upstream has a value, I would expect the throttle based on
that key (2x_zone_from_upstream) to take effect, not 2x_zone whose key is
empty.

I would really appreciate any help explaining my misunderstanding or advice
with how better to implement what I'm trying to do.

Thanks,
Jared

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,281034,281034#msg-281034



More information about the nginx mailing list