$upstream_http_NAME and $sent_http_NAME vars not available in certain scopes

Wed Mar 30 11:11:45 UTC 2016

Hi all

I am developing a proxy service which uses NGINX to reverse proxy, kind of
CDN-like but very specific to our needs. During this. I have hit an issue
which I *think* is a bug but wanted to ask in case anyone can point to a
solution or some reading I can do. The problem I have is this:

$upstream_http_NAME and $sent_http_NAME variables seem to be
unpopulated/blank at some points in my config when other embedded variables
*are* populated. This is probably best illustrated via an example, so here's
a simplified test case I created:

user nginx;
worker_processes auto;
worker_priority -15;
worker_rlimit_nofile 50000;

events {
    # worker_connections benefits from a large value in that it reduces
error counts
    worker_connections 20000;
    multi_accept on;
}

http {

# for dynamic upstreams
resolver 8.8.8.8;

	# Default includes
	include       /etc/nginx/current/mime.types;
	default_type  application/octet-stream;
	include /etc/nginx/current/proxy.conf;

	# Tuning options - these are mainly quite GTM-specific
	server_tokens off;
	keepalive_requests 1024;
	keepalive_timeout 120s 120s;
	sendfile on;
	tcp_nodelay on;
	tcp_nopush on;
	client_header_timeout 5s;
	open_file_cache max=16384 inactive=600s;
	open_file_cache_valid 600s;
	open_file_cache_min_uses 0;
	open_file_cache_errors on;
	output_buffers 64 128k;

# NEW - AIO
aio on;
directio 512;

	# For small files and heavy load, this gives ~5-6x greater throughput
(avoids swamping workers with one request)
	postpone_output 0;
	reset_timedout_connection on;
	send_timeout 3s;
	sendfile_max_chunk 1m;
	large_client_header_buffers 8 8k;
	connection_pool_size 4096;

	# client_body_buffer_size - Sets buffer size for reading client request
body. In case the request body is larger than the buffer, the whole body or
only its part is written to a temporary file
	client_body_buffer_size 8k;
	client_header_buffer_size 8k;

	# client_max_body_size - Sets the maximum allowed size of the client
request body, specified in the “Content-Length” request header field. If the
size in a request exceeds the configured value, the 413 (Request Entity Too
Large) error is returned to the client
	client_max_body_size 8m;

	# We need to increase the hash bucket size as the R53 names are long!
	server_names_hash_bucket_size 128;

	# Same for proxy_headers_hash_max_size and proxy_headers_hash_bucket_size
	proxy_headers_hash_max_size 4096;
	proxy_headers_hash_bucket_size 1024;

# Logging
	# NOTE: $host may need to in fact be $hostname
	log_format standard '"$remote_addr" "$time_iso8601" "$request_method"
"$scheme" "$host" "$request_uri" "$server_protocol" "$status" "$bytes_sent"
"$http_referer" "$http_user_agent" "$ssl_protocol" "$ssl_cipher"
"$ssl_server_name" "$ssl_session_reused"';
	access_log /var/log/nginx/main-access.log standard;
	error_log /var/log/nginx/main-error.log warn;

	recursive_error_pages on;

# GeoIP config
	# This appears to need an absolute path, despite the docs suggesting it
doesn't.
	# Path is defined in bake script so changes to that will break this
#	geoip_country /usr/local/GeoIP.dat;
geoip_city /var/lib/GeoIP/GeoLiteCity.dat;
	#geoip_proxy 0.0.0.0/0;
	#geoip_proxy_recursive on;

# Proxy global configuration
	# NOTES:
	# proxy_cache_path is an http scope (global) directive
	# keys_zone=shared_cache:XXXXm; denotes the amount of RAM to allow for
cache index, 1MB ~7k-8k cached objects - exceeding the number of cached
objects possible due to index size results in LRU invocation
	proxy_cache_path /mnt/gtm_cache_data levels=1:2 use_temp_path=on
keys_zone=shared_cache:256m inactive=1440m;
	proxy_temp_path /mnt/gtm_cache_temp;

	# NGINX recommends HTTP 1.1 for keepalive, proxied conns. (which we use)
	proxy_http_version 1.1;
	# NGINX recommends clearing the connection request header for keepalive
http 1.1 conns
	proxy_set_header Connection "";

	# Conditions under which we want to try then next (if there is one)
upstream server in the list & timeouts
	proxy_next_upstream error timeout invalid_header http_500 http_502 http_503
http_504;
	proxy_next_upstream_timeout 5s;
	proxy_next_upstream_tries 3;

ssl_session_cache shared:global_ssl_cache:128m;

ssl_session_timeout 120m;
ssl_session_tickets on;
#ssl_session_ticket_key /etc/nginx/current/tls/session/tkt.key;

# TEST CASE:
# Set up a DNS or hosts file entry to point to your NGINX instance running
this config
# Hit this with URL:
https://<HOSTNAME>/response-headers?Content-Type=text%2Fplain%3B+charset%3DUTF-8&via=1.1%20httpbin3&tester=hello

	# To try to check if we can work around the problem below (vars not
existing (?) at time of use/need), we'll try to copy the var via a map
	map $upstream_http_via $copy_map_upstream_http_via {
		default $upstream_http_via;
	}

	upstream origin {
		server httpbin.org:443 resolve;
	}

	# Generic/common server for listen port configs
	server {

		# TMP removing fastopen=None backlog=-1 as the directives don't work!
		listen *:80 so_keepalive=120s:30s:20 default_server;
		listen *:443 ssl http2 reuseport deferred so_keepalive=120s:30s:20
default_server;

		# This cert & key will never actually be used but are needed to allow the
:443 operation - without them the connection will be closed
		ssl_certificate     /etc/nginx/current/tls/certs/default.crt;
		ssl_certificate_key /etc/nginx/current/tls/private/default.key;

		ssl_protocols TLSv1 TLSv1.1 TLSv1.2;

		location / {

			# To try to check if we can work around the problem below (vars not
existing (?) at time of use/need), we'll try to copy the var via a set
			set $copy_set_upstream_http_via $upstream_http_via;

			more_set_headers "SENT_HTTP_VIA: $sent_http_via";
			more_set_headers "HTTP_VIA: $http_via";
			more_set_headers "UPSTREAM_HTTP_VIA: $upstream_http_via";

			# For ref, the upstream is sending e.g. "via: 1.1 string"

			# Problem: These do not match, $upstream_http_via appears not to be
populated at this point
#                       if ( $sent_http_via ~* "^[0-9]\.[0-9]\ .+$" ) {
#			if ($upstream_http_via ~* "^[0-9]\.[0-9]\ .+") {
#			if ($sent_http_via ~* "^[0-9]\.[0-9]\ .+") {
#			if ($upstream_http_via) {

			# This does match - for obvious reasons
			if ($upstream_http_via ~* ".*") {

			# Problem: $upstream_http_via appears not to be populated at this
point...
				set $via_comp "$upstream_http_via 1.y BLAH";
				more_set_headers "IF_VIA_COMP: Value is $via_comp";

				# Just to demo the string concat - we'll use a different embedded var
				set $test_comp "$ssl_protocol BLAH";
                               	more_set_headers "IF_TEST_COMP: Value is
$test_comp";

				# Does the map-copied var exist?
				set $via_comp_copy_map "$copy_map_upstream_http_via 1.y BLAH";
                                more_set_headers "IF_VIA_COMP_COPY_MAP:
Value is $via_comp_copy_map";

				# Does the set-copied var exist?
				set $via_comp_copy_set "$copy_set_upstream_http_via 1.y BLAH";
                                more_set_headers "IF_VIA_COMP_COPY_SET:
Value is $via_comp_copy_set";

				# Does a different $upstream_http_X var work? - NO
				set $alt_comp "$upstream_http_tester 1.y BLAH";
                                more_set_headers "IF_ALT_COMP: Value is
$alt_comp";

			# ...but $upstream_http_via IS populated at this point
				more_set_headers "UPSTREAM_HTTP_VIA_IF: $upstream_http_via";
				more_set_headers "HTTP_VIA_IF: $http_via";
				more_set_headers "SENT_HTTP_VIA_IF: $sent_http_via";
			}

			proxy_pass https://origin;
		}

# END TEST CASE

	}
}

(Sorry, couldn't figure out how to markup the config)

The response headers from a request to this NGIX instance is e.g.:

access-control-allow-credentials:true
access-control-allow-origin:*
content-length:161
content-type:text/plain; charset=UTF-8
date:Wed, 30 Mar 2016 10:31:55 GMT
if_alt_comp:Value is  1.y BLAH
if_test_comp:Value is TLSv1.2 BLAH
if_via_comp:Value is  1.y BLAH
if_via_comp_copy_map:Value is  1.y BLAH
if_via_comp_copy_set:Value is  1.y BLAH
sent_http_via:1.1 httpbin3
sent_http_via_if:1.1 httpbin3
server:nginx
status:200
tester:hello
upstream_http_via:1.1 httpbin3
upstream_http_via_if:1.1 httpbin3
via:1.1 httpbin3

So you can see from the section:
if_alt_comp:Value is  1.y BLAH
if_test_comp:Value is TLSv1.2 BLAH
if_via_comp:Value is  1.y BLAH
if_via_comp_copy_map:Value is  1.y BLAH
if_via_comp_copy_set:Value is  1.y BLAH

That there's a blank space where the value from $upstream_http_via or
$sent_http_via should be.

So my questions are:
Can anyone see something I have done wrong?
Is this expected behaviour (if yes, why? Seems strange some vars behave
differently)
Does anyone have a workaround or solution?

Many thanks in advance if anyone can offer any help.

Cheers
Neil

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,265734,265734#msg-265734