nginx-devel August 2021

nginx-devel@nginx.org

18 participants
58 discussions

[PATCH] Add optional "mp4_exact_start" nginx config off/on to show video between keyframes

by Tracey Jaquith

# HG changeset patch # User Tracey Jaquith <tracey(a)archive.org> # Date 1623797180 0 # Tue Jun 15 22:46:20 2021 +0000 # Node ID 1879d49fe0cf739f48287b5a38a83d3a1adab939 # Parent 5f765427c17ac8cf753967387562201cf4f78dc4 Add optional "mp4_exact_start" nginx config off/on to show video between keyframes. archive.org has been using mod_h264_streaming with a similar "exact start" patch from me since 2013. We just moved to nginx mp4 module and are using this patch. The technique is to find the video keyframe just before the desired "start" time, and send that down the wire so video playback can start immediately. Next calculate how many video samples are between the keyframe and desired "start" time and update the STTS atom where those samples move the duration from (typically) 1001 to 1. This way, initial unwanted video frames play at ~1/30,000s -- so visually the video & audio start playing immediately. You can see an example before/after here (nginx binary built with mp4 module + patch): https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Corre… https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Corre… Tested on linux and macosx. (this is me: https://github.com/traceypooh ) diff -r 5f765427c17a -r 1879d49fe0cf src/http/modules/ngx_http_mp4_module.c --- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300 +++ b/src/http/modules/ngx_http_mp4_module.c Tue Jun 15 22:46:20 2021 +0000 @@ -43,6 +43,7 @@ typedef struct { size_t buffer_size; size_t max_buffer_size; + ngx_flag_t exact_start; } ngx_http_mp4_conf_t; @@ -340,6 +341,13 @@ offsetof(ngx_http_mp4_conf_t, max_buffer_size), NULL }, + { ngx_string("mp4_exact_start"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_mp4_conf_t, exact_start), + NULL }, + ngx_null_command }; @@ -2156,6 +2164,83 @@ static ngx_int_t +ngx_http_mp4_exact_start_video(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak) +{ + uint32_t n, speedup_samples, current_count; + ngx_uint_t sample_keyframe, start_sample_exact; + ngx_mp4_stts_entry_t *entry, *entries_array; + ngx_buf_t *data; + + data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf; + + // Find the keyframe just before the desired start time - so that we can emit an mp4 + // where the first frame is a keyframe. We'll "speed up" the first frames to 1000x + // normal speed (typically), so they won't be noticed. But this way, perceptively, + // playback of the _video_ track can start immediately + // (and not have to wait until the keyframe _after_ the desired starting time frame). + start_sample_exact = trak->start_sample; + for (n = 0; n < trak->sync_samples_entries; n++) { + // each element of array is the sample number of a keyframe + // sync samples starts from 1 -- so subtract 1 + sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4)) - 1; + if (sample_keyframe <= trak->start_sample) { + start_sample_exact = sample_keyframe; + } + if (sample_keyframe >= trak->start_sample) { + break; + } + } + + if (start_sample_exact < trak->start_sample) { + // We're going to prepend an entry with duration=1 for the frames we want to "not see". + // MOST of the time (eg: constant video framerate), + // we're taking a single element entry array and making it two. + speedup_samples = trak->start_sample - start_sample_exact; + + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact trak start_sample move %l to %l (speed up %d samples)\n", + trak->start_sample, start_sample_exact, speedup_samples); + + entries_array = ngx_palloc(mp4->request->pool, + (1 + trak->time_to_sample_entries) * sizeof(ngx_mp4_stts_entry_t)); + if (entries_array == NULL) { + return NGX_ERROR; + } + entry = &(entries_array[1]); + ngx_memcpy(entry, (ngx_mp4_stts_entry_t *)data->pos, + trak->time_to_sample_entries * sizeof(ngx_mp4_stts_entry_t)); + + current_count = ngx_mp4_get_32value(entry->count); + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split in 2 video STTS entry from count:%d", current_count); + + if (current_count <= speedup_samples) { + return NGX_ERROR; + } + + ngx_mp4_set_32value(entry->count, current_count - speedup_samples); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split new[1]: count:%d duration:%d", + ngx_mp4_get_32value(entry->count), + ngx_mp4_get_32value(entry->duration)); + entry--; + ngx_mp4_set_32value(entry->count, speedup_samples); + ngx_mp4_set_32value(entry->duration, 1); + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "exact split new[0]: count:%d duration:1", + ngx_mp4_get_32value(entry->count)); + + data->pos = (u_char *) entry; + trak->time_to_sample_entries++; + trak->start_sample = start_sample_exact; + data->last = (u_char *) (entry + trak->time_to_sample_entries); + } + + return NGX_OK; +} + + +static ngx_int_t ngx_http_mp4_crop_stts_data(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_uint_t start) { @@ -2164,6 +2249,8 @@ ngx_buf_t *data; ngx_uint_t start_sample, entries, start_sec; ngx_mp4_stts_entry_t *entry, *end; + ngx_http_mp4_conf_t *conf; + if (start) { start_sec = mp4->start; @@ -2238,6 +2325,10 @@ "start_sample:%ui, new count:%uD", trak->start_sample, count - rest); + conf = ngx_http_get_module_loc_conf(mp4->request, ngx_http_mp4_module); + if (conf->exact_start) { + ngx_http_mp4_exact_start_video(mp4, trak); + } } else { ngx_mp4_set_32value(entry->count, rest); data->last = (u_char *) (entry + 1); @@ -3590,6 +3681,7 @@ conf->buffer_size = NGX_CONF_UNSET_SIZE; conf->max_buffer_size = NGX_CONF_UNSET_SIZE; + conf->exact_start = NGX_CONF_UNSET; return conf; }

2 months

test suite failure with 1.20.1

by Hugo Lefeuvre

Hi, I am trying to run the test suite, but it seems that, no matter how I build Nginx, it systematically fails. It seems that, most (all?) of the time, tests fail because Nginx returns 403 error codes, e.g.: ./ssi_waited.t ............................. 1/3 # Failed test 'waited non-active' # at ./ssi_waited.t line 60. # 'HTTP/1.1 403 Forbidden # Server: nginx/1.21.0 # Date: Sat, 03 Jul 2021 08:06:00 GMT # Content-Type: text/html # Connection: close # # <html> # <head><title>403 Forbidden</title></head> # <body> # <center><h1>403 Forbidden</h1></center> # <hr><center>nginx/1.21.0</center> # </body> # </html> # ' # doesn't match '(?^m:^xFIRSTxWAITEDxSECONDx$)' The runtime configuration is the default one from nginx-1.20.1.tar.gz (conf/nginx.conf). I must be doing something wrong with the build or run time configuration, but I cannot pinpoint what. Any idea? Best, Hugo -- Hugo Lefeuvre (hle) | www.owl.eu.com RSA4096_ 360B 03B3 BF27 4F4D 7A3F D5E8 14AA 1EB8 A247 3DFD ed25519_ 37B2 6D38 0B25 B8A2 6B9F 3A65 A36F 5357 5F2D DC4C

7 months

[PATCH] Avoid unnecessary restriction on nohash http variables

by Alexey Radkov

# HG changeset patch # User Alexey Radkov <alexey.radkov(a)gmail.com> # Date 1629395487 -10800 # Thu Aug 19 20:51:27 2021 +0300 # Node ID a1065b2252855730ed8e5368c88fe41a7ff5a698 # Parent 13d0c1d26d47c203b1874ca1ffdb7a9ba7fd2d77 Avoid unnecessary restriction on nohash http variables. When I use variables with long names albeit being tagged as NGX_HTTP_VARIABLE_NOHASH, Nginx says "could not build variables_hash, you should increase variables_hash_bucket_size: 64". It seems that this is an unnecessary restriction, as soon as the hash gets only built for variables with names[n].key.data == NULL (note that other pieces in ngx_hash_init() where the macro NGX_HASH_ELT_SIZE is used, are always guarded with this condition). This fix puts this same condition into the only unguarded piece: when testing against the hash_bucket_size. The issue arises after assignment of key[n].key.data = NULL without symmetric assignment of key[n].key.len in ngx_http_variables_init_vars(): after this, the key[n].key comes to an inconsistent state. Perhaps this was made intentionally, as hash initialization in other places seems to follow the same pattern (for instance, see how ngx_hash_init() gets called from ngx_http_upstream_hide_headers_hash()). Without this fix, I must put in the config "variables_hash_bucket_size 128;" even if the long-named variables are nohash. diff -r 13d0c1d26d47 -r a1065b225285 src/core/ngx_hash.c --- a/src/core/ngx_hash.c Fri Aug 13 03:57:47 2021 -0400 +++ b/src/core/ngx_hash.c Thu Aug 19 20:51:27 2021 +0300 @@ -274,6 +274,9 @@ } for (n = 0; n < nelts; n++) { + if (names[n].key.data == NULL) { + continue; + } if (hinit->bucket_size < NGX_HASH_ELT_SIZE(&names[n]) + sizeof(void *)) { ngx_log_error(NGX_LOG_EMERG, hinit->pool->log, 0,

9 months, 3 weeks

[PATCH] Added support for proxying managesieve protocol

by Sander Hoentjen

# HG changeset patch # User Sander Hoentjen <shoentjen(a)antagonist.nl> # Date 1586369831 -7200 # Wed Apr 08 20:17:11 2020 +0200 # Node ID f1dffaf619688aaab90caf31781ebe27c3f79598 # Parent 0cb942c1c1aa98118076e72e0b89940e85e6291c Added support for proxying managesieve protocol Sieve is a widely used protocol to implement server-side filtering. Some servers and clients that support this are listed at <http://sieve.info/>. This commit is tested with Dovecot as the server and Roundcube as the client, but I don't see a readon why it not would work with others. It is a nice addition to nginx because it adds the possibility to add server side filtering to setups that use IMAP behind an nginx proxy. Fixes https://trac.nginx.org/nginx/ticket/1697 diff -r 0cb942c1c1aa -r f1dffaf61968 auto/modules --- a/auto/modules Fri Mar 13 02:12:10 2020 +0300 +++ b/auto/modules Wed Apr 08 20:17:11 2020 +0200 @@ -965,6 +965,15 @@ . auto/module fi + if [ $MAIL_SIEVE = YES ]; then + ngx_module_name=ngx_mail_sieve_module + ngx_module_deps=src/mail/ngx_mail_sieve_module.h + ngx_module_srcs="src/mail/ngx_mail_sieve_module.c \ + src/mail/ngx_mail_sieve_handler.c" + + . auto/module + fi + if [ $MAIL_SMTP = YES ]; then ngx_module_name=ngx_mail_smtp_module ngx_module_deps=src/mail/ngx_mail_smtp_module.h diff -r 0cb942c1c1aa -r f1dffaf61968 auto/options --- a/auto/options Fri Mar 13 02:12:10 2020 +0300 +++ b/auto/options Wed Apr 08 20:17:11 2020 +0200 @@ -112,6 +112,7 @@ MAIL_SSL=NO MAIL_POP3=YES MAIL_IMAP=YES +MAIL_SIEVE=YES MAIL_SMTP=YES STREAM=NO @@ -305,6 +306,8 @@ ;; --without-mail_pop3_module) MAIL_POP3=NO ;; --without-mail_imap_module) MAIL_IMAP=NO ;; + --without-mail_sieve_module) + MAIL_SIEVE=NO ;; --without-mail_smtp_module) MAIL_SMTP=NO ;; --with-stream) STREAM=YES ;; @@ -517,11 +520,12 @@ --without-http disable HTTP server --without-http-cache disable HTTP cache - --with-mail enable POP3/IMAP4/SMTP proxy module - --with-mail=dynamic enable dynamic POP3/IMAP4/SMTP proxy module + --with-mail enable POP3/IMAP4/SIEVE/SMTP proxy module + --with-mail=dynamic enable dynamic POP3/IMAP4/SIEVE/SMTP proxy module --with-mail_ssl_module enable ngx_mail_ssl_module --without-mail_pop3_module disable ngx_mail_pop3_module --without-mail_imap_module disable ngx_mail_imap_module + --without-mail_sieve_module disable ngx_mail_sieve_module --without-mail_smtp_module disable ngx_mail_smtp_module --with-stream enable TCP/UDP proxy module diff -r 0cb942c1c1aa -r f1dffaf61968 contrib/vim/syntax/nginx.vim --- a/contrib/vim/syntax/nginx.vim Fri Mar 13 02:12:10 2020 +0300 +++ b/contrib/vim/syntax/nginx.vim Wed Apr 08 20:17:11 2020 +0200 @@ -571,6 +571,9 @@ syn keyword ngxDirective contained session_log_format syn keyword ngxDirective contained session_log_zone syn keyword ngxDirective contained set_real_ip_from +syn keyword ngxDirective contained sieve_auth +syn keyword ngxDirective contained sieve_capabilities +syn keyword ngxDirective contained sieve_client_buffer syn keyword ngxDirective contained slice syn keyword ngxDirective contained smtp_auth syn keyword ngxDirective contained smtp_capabilities diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail.h --- a/src/mail/ngx_mail.h Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail.h Wed Apr 08 20:17:11 2020 +0200 @@ -102,6 +102,7 @@ #define NGX_MAIL_POP3_PROTOCOL 0 #define NGX_MAIL_IMAP_PROTOCOL 1 #define NGX_MAIL_SMTP_PROTOCOL 2 +#define NGX_MAIL_SIEVE_PROTOCOL 3 typedef struct ngx_mail_protocol_s ngx_mail_protocol_t; @@ -154,6 +155,21 @@ typedef enum { + ngx_sieve_start = 0, + ngx_sieve_starttls, + ngx_sieve_auth_login_username, + ngx_sieve_auth_login_password, + ngx_sieve_auth_plain, + ngx_sieve_auth_cram_md5, + ngx_sieve_auth_external, + ngx_sieve_login, + ngx_sieve_login_capabilities, + ngx_sieve_user, + ngx_sieve_passwd +} ngx_sieve_state_e; + + +typedef enum { ngx_smtp_start = 0, ngx_smtp_auth_login_username, ngx_smtp_auth_login_password, @@ -227,7 +243,7 @@ ngx_uint_t login_attempt; - /* used to parse POP3/IMAP/SMTP command */ + /* used to parse POP3/IMAP/SIEVE/SMTP command */ ngx_uint_t state; u_char *cmd_start; @@ -271,6 +287,14 @@ #define NGX_IMAP_AUTHENTICATE 7 +#define NGX_SIEVE_LOGOUT 1 +#define NGX_SIEVE_CAPABILITY 2 +#define NGX_SIEVE_NOOP 3 +#define NGX_SIEVE_STARTTLS 4 +#define NGX_SIEVE_NEXT 5 +#define NGX_SIEVE_AUTHENTICATE 6 + + #define NGX_SMTP_HELO 1 #define NGX_SMTP_EHLO 2 #define NGX_SMTP_AUTH 3 @@ -383,7 +407,7 @@ ngx_int_t ngx_mail_auth_login_password(ngx_mail_session_t *s, ngx_connection_t *c); ngx_int_t ngx_mail_auth_cram_md5_salt(ngx_mail_session_t *s, - ngx_connection_t *c, char *prefix, size_t len); + ngx_connection_t *c, char *prefix, size_t len, ngx_uint_t quote); ngx_int_t ngx_mail_auth_cram_md5(ngx_mail_session_t *s, ngx_connection_t *c); ngx_int_t ngx_mail_auth_external(ngx_mail_session_t *s, ngx_connection_t *c, ngx_uint_t n); diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_auth_http_module.c --- a/src/mail/ngx_mail_auth_http_module.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_auth_http_module.c Wed Apr 08 20:17:11 2020 +0200 @@ -533,6 +533,11 @@ + sizeof(CRLF) - 1; break; + case NGX_MAIL_SIEVE_PROTOCOL: + size = sizeof("BYE \"\"") - 1 + len + + sizeof(CRLF) - 1; + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ ctx->err = ctx->errmsg; continue; @@ -559,11 +564,27 @@ *p++ = 'N'; *p++ = 'O'; *p++ = ' '; break; + case NGX_MAIL_SIEVE_PROTOCOL: + p = ngx_cpymem(p, s->tag.data, s->tag.len); + *p++ = 'B'; *p++ = 'Y'; *p++ = 'E'; *p++ = ' '; *p++ = '"'; + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ break; } p = ngx_cpymem(p, ctx->header_start, len); + + switch (s->protocol) { + + case NGX_MAIL_SIEVE_PROTOCOL: + *p++ = '"'; + break; + + default: + break; + } + *p++ = CR; *p++ = LF; ctx->err.len = p - ctx->err.data; diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_handler.c --- a/src/mail/ngx_mail_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -520,28 +520,39 @@ ngx_int_t ngx_mail_auth_cram_md5_salt(ngx_mail_session_t *s, ngx_connection_t *c, - char *prefix, size_t len) + char *prefix, size_t len, ngx_uint_t quote) { u_char *p; ngx_str_t salt; - ngx_uint_t n; + ngx_uint_t n = 0; + if (quote) { + len += 2; + } p = ngx_pnalloc(c->pool, len + ngx_base64_encoded_length(s->salt.len) + 2); if (p == NULL) { return NGX_ERROR; } - salt.data = ngx_cpymem(p, prefix, len); + if (quote) { + len -= 2; + salt.data = ngx_cpymem(p, "\"", 1); + n++; + } + salt.data = ngx_cpymem(p+ n, prefix, len); s->salt.len -= 2; ngx_encode_base64(&salt, &s->salt); s->salt.len += 2; - n = len + salt.len; + n += len + salt.len; + if (quote) { + p[n++] = '"'; + } p[n++] = CR; p[n++] = LF; s->out.len = n; - s->out.data = p; + s->out.data = p--; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_imap_handler.c --- a/src/mail/ngx_mail_imap_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_imap_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -397,7 +397,7 @@ } } - if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2) == NGX_OK) { + if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2, 0) == NGX_OK) { s->mail_state = ngx_imap_auth_cram_md5; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_parse.c --- a/src/mail/ngx_mail_parse.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_parse.c Wed Apr 08 20:17:11 2020 +0200 @@ -11,6 +11,7 @@ #include <ngx_mail.h> #include <ngx_mail_pop3_module.h> #include <ngx_mail_imap_module.h> +#include <ngx_mail_sieve_module.h> #include <ngx_mail_smtp_module.h> @@ -620,6 +621,364 @@ ngx_int_t +ngx_mail_sieve_parse_command(ngx_mail_session_t *s) +{ + u_char ch, *p, *c; + ngx_str_t *arg; + enum { + sw_start = 0, + sw_spaces_before_argument, + sw_argument, + sw_backslash, + sw_literal, + sw_no_sync_literal_argument, + sw_start_literal_argument, + sw_literal_argument, + sw_end_literal_argument, + sw_almost_done + } state; + + state = s->state; + + for (p = s->buffer->pos; p < s->buffer->last; p++) { + ch = *p; + + switch (state) { + + /* SIEVE command */ + case sw_start: + if (ch == ' ' || ch == CR || ch == LF) { + + c = s->buffer->start; + + switch (p - c) { + + case 4: + if ((c[0] == 'N' || c[0] == 'n') + && (c[1] == 'O'|| c[1] == 'o') + && (c[2] == 'O'|| c[2] == 'o') + && (c[3] == 'P'|| c[3] == 'p')) + { + s->command = NGX_SIEVE_NOOP; + + } else { + goto invalid; + } + break; + + case 6: + if ((c[0] == 'L'|| c[0] == 'l') + && (c[1] == 'O'|| c[1] == 'o') + && (c[2] == 'G'|| c[2] == 'g') + && (c[3] == 'O'|| c[3] == 'o') + && (c[4] == 'U'|| c[4] == 'u') + && (c[5] == 'T'|| c[5] == 't')) + { + s->command = NGX_SIEVE_LOGOUT; + + } else { + goto invalid; + } + break; + +#if (NGX_MAIL_SSL) + case 8: + if ((c[0] == 'S'|| c[0] == 's') + && (c[1] == 'T'|| c[1] == 't') + && (c[2] == 'A'|| c[2] == 'a') + && (c[3] == 'R'|| c[3] == 'r') + && (c[4] == 'T'|| c[4] == 't') + && (c[5] == 'T'|| c[5] == 't') + && (c[6] == 'L'|| c[6] == 'l') + && (c[7] == 'S'|| c[7] == 's')) + { + s->command = NGX_SIEVE_STARTTLS; + + } else { + goto invalid; + } + break; +#endif + + case 10: + if ((c[0] == 'C'|| c[0] == 'c') + && (c[1] == 'A'|| c[1] == 'a') + && (c[2] == 'P'|| c[2] == 'p') + && (c[3] == 'A'|| c[3] == 'a') + && (c[4] == 'B'|| c[4] == 'b') + && (c[5] == 'I'|| c[5] == 'i') + && (c[6] == 'L'|| c[6] == 'l') + && (c[7] == 'I'|| c[7] == 'i') + && (c[8] == 'T'|| c[8] == 't') + && (c[9] == 'Y'|| c[9] == 'y')) + { + s->command = NGX_SIEVE_CAPABILITY; + + } else { + goto invalid; + } + break; + + case 12: + if ((c[0] == 'A'|| c[0] == 'a') + && (c[1] == 'U'|| c[1] == 'u') + && (c[2] == 'T'|| c[2] == 't') + && (c[3] == 'H'|| c[3] == 'h') + && (c[4] == 'E'|| c[4] == 'e') + && (c[5] == 'N'|| c[5] == 'n') + && (c[6] == 'T'|| c[6] == 't') + && (c[7] == 'I'|| c[7] == 'i') + && (c[8] == 'C'|| c[8] == 'c') + && (c[9] == 'A'|| c[9] == 'a') + && (c[10] == 'T'|| c[10] == 't') + && (c[11] == 'E'|| c[11] == 'e')) + { + s->command = NGX_SIEVE_AUTHENTICATE; + + } else { + goto invalid; + } + break; + + default: + goto invalid; + } + + switch (ch) { + case ' ': + state = sw_spaces_before_argument; + break; + case CR: + state = sw_almost_done; + break; + case LF: + goto done; + } + break; + } + + if ((ch < 'A' || ch > 'Z') && (ch < 'a' || ch > 'z')) { + goto invalid; + } + + break; + + case sw_spaces_before_argument: + switch (ch) { + case ' ': + break; + case CR: + state = sw_almost_done; + s->arg_end = p; + break; + case LF: + s->arg_end = p; + goto done; + case '"': + if (s->args.nelts <= 2) { + s->quoted = 1; + s->arg_start = p + 1; + state = sw_argument; + break; + } + goto invalid; + case '{': + if (s->args.nelts <= 2) { + state = sw_literal; + break; + } + goto invalid; + default: + if (s->args.nelts <= 2) { + s->arg_start = p; + state = sw_argument; + break; + } + goto invalid; + } + break; + + case sw_argument: + if (ch == ' ' && s->quoted) { + break; + } + + switch (ch) { + case '"': + if (!s->quoted) { + break; + } + s->quoted = 0; + /* fall through */ + case ' ': + case CR: + case LF: + arg = ngx_array_push(&s->args); + if (arg == NULL) { + return NGX_ERROR; + } + arg->len = p - s->arg_start; + arg->data = s->arg_start; + s->arg_start = NULL; + + switch (ch) { + case '"': + case ' ': + state = sw_spaces_before_argument; + break; + case CR: + state = sw_almost_done; + break; + case LF: + goto done; + } + break; + case '\\': + if (s->quoted) { + s->backslash = 1; + state = sw_backslash; + } + break; + } + break; + + case sw_backslash: + switch (ch) { + case CR: + case LF: + goto invalid; + default: + state = sw_argument; + } + break; + + case sw_literal: + if (ch >= '0' && ch <= '9') { + s->literal_len = s->literal_len * 10 + (ch - '0'); + break; + } + if (ch == '}') { + state = sw_start_literal_argument; + break; + } + if (ch == '+') { + state = sw_no_sync_literal_argument; + break; + } + goto invalid; + + case sw_no_sync_literal_argument: + if (ch == '}') { + s->no_sync_literal = 1; + state = sw_start_literal_argument; + break; + } + goto invalid; + + case sw_start_literal_argument: + switch (ch) { + case CR: + break; + case LF: + s->buffer->pos = p + 1; + s->arg_start = p + 1; + if (s->no_sync_literal == 0) { + s->state = sw_literal_argument; + return NGX_SIEVE_NEXT; + } + state = sw_literal_argument; + s->no_sync_literal = 0; + break; + default: + goto invalid; + } + break; + + case sw_literal_argument: + if (s->literal_len && --s->literal_len) { + break; + } + + arg = ngx_array_push(&s->args); + if (arg == NULL) { + return NGX_ERROR; + } + arg->len = p + 1 - s->arg_start; + arg->data = s->arg_start; + s->arg_start = NULL; + state = sw_end_literal_argument; + + break; + + case sw_end_literal_argument: + switch (ch) { + case '{': + if (s->args.nelts <= 2) { + state = sw_literal; + break; + } + goto invalid; + case CR: + state = sw_almost_done; + break; + case LF: + goto done; + default: + state = sw_spaces_before_argument; + break; + } + break; + + case sw_almost_done: + switch (ch) { + case LF: + goto done; + default: + goto invalid; + } + } + } + + s->buffer->pos = p; + s->state = state; + + return NGX_AGAIN; + +done: + + s->buffer->pos = p + 1; + + if (s->arg_start) { + arg = ngx_array_push(&s->args); + if (arg == NULL) { + return NGX_ERROR; + } + arg->len = s->arg_end - s->arg_start; + arg->data = s->arg_start; + + s->arg_start = NULL; + s->cmd_start = NULL; + s->quoted = 0; + s->no_sync_literal = 0; + s->literal_len = 0; + } + + s->state = (s->command != NGX_SIEVE_AUTHENTICATE) ? sw_start : sw_spaces_before_argument; + + return NGX_OK; + +invalid: + + s->state = sw_start; + s->quoted = 0; + s->no_sync_literal = 0; + s->literal_len = 0; + + return NGX_MAIL_PARSE_INVALID_COMMAND; +} + + +ngx_int_t ngx_mail_smtp_parse_command(ngx_mail_session_t *s) { u_char ch, *p, *c, c0, c1, c2, c3; diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_pop3_handler.c --- a/src/mail/ngx_mail_pop3_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_pop3_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -492,7 +492,7 @@ return NGX_MAIL_PARSE_INVALID_COMMAND; } - if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2) == NGX_OK) { + if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2, 0) == NGX_OK) { s->mail_state = ngx_pop3_auth_cram_md5; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_proxy_module.c --- a/src/mail/ngx_mail_proxy_module.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_proxy_module.c Wed Apr 08 20:17:11 2020 +0200 @@ -24,6 +24,7 @@ static void ngx_mail_proxy_block_read(ngx_event_t *rev); static void ngx_mail_proxy_pop3_handler(ngx_event_t *rev); static void ngx_mail_proxy_imap_handler(ngx_event_t *rev); +static void ngx_mail_proxy_sieve_handler(ngx_event_t *rev); static void ngx_mail_proxy_smtp_handler(ngx_event_t *rev); static void ngx_mail_proxy_dummy_handler(ngx_event_t *ev); static ngx_int_t ngx_mail_proxy_read_response(ngx_mail_session_t *s, @@ -173,6 +174,11 @@ s->mail_state = ngx_imap_start; break; + case NGX_MAIL_SIEVE_PROTOCOL: + p->upstream.connection->read->handler = ngx_mail_proxy_sieve_handler; + s->mail_state = ngx_sieve_start; + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ p->upstream.connection->read->handler = ngx_mail_proxy_smtp_handler; s->mail_state = ngx_smtp_start; @@ -446,6 +452,144 @@ static void +ngx_mail_proxy_sieve_handler(ngx_event_t *rev) +{ + u_char *p; + ngx_int_t rc; + ngx_str_t line; + ngx_str_t userpass; + ngx_str_t b64_userpass; + ngx_connection_t *c; + ngx_mail_session_t *s; + ngx_mail_proxy_conf_t *pcf; + + ngx_log_debug0(NGX_LOG_DEBUG_MAIL, rev->log, 0, + "mail proxy sieve auth handler"); + + c = rev->data; + s = c->data; + + if (rev->timedout) { + ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, + "upstream timed out"); + c->timedout = 1; + ngx_mail_proxy_internal_server_error(s); + return; + } + + rc = ngx_mail_proxy_read_response(s, s->mail_state); + + if (rc == NGX_AGAIN) { + return; + } + + if (rc == NGX_ERROR) { + ngx_mail_proxy_upstream_error(s); + return; + } + + switch (s->mail_state) { + + case ngx_sieve_start: + ngx_log_debug0(NGX_LOG_DEBUG_MAIL, rev->log, 0, + "mail proxy send login"); + + s->connection->log->action = "sending LOGIN command to upstream"; + + line.len = s->tag.len + sizeof("AUTHENTICATE ") - 1 + + 1 + NGX_SIZE_T_LEN + 1 + 2; + line.data = ngx_pnalloc(c->pool, line.len); + if (line.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + + userpass.len = s->login.len + s->passwd.len + 2; + userpass.data = ngx_pnalloc(c->pool, userpass.len); + if (userpass.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + b64_userpass.len = ngx_base64_encoded_length(userpass.len) + 2; + b64_userpass.data = ngx_pnalloc(c->pool, b64_userpass.len); + if (b64_userpass.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + + p = userpass.data; + *p++ = '\0'; + p = ngx_cpymem(p, s->login.data, s->login.len); + *p++ = '\0'; + p = ngx_cpymem(p, s->passwd.data, s->passwd.len); + + ngx_encode_base64(&b64_userpass, &userpass); + line.len = ngx_sprintf(line.data, "AUTHENTICATE \"PLAIN\" \"%V\"" CRLF, + &b64_userpass) + - line.data; + + s->mail_state = ngx_sieve_login; + break; + + case ngx_sieve_login: + ngx_log_debug0(NGX_LOG_DEBUG_MAIL, rev->log, 0, + "mail proxy send capabilities"); + + s->connection->log->action = "sending CAPABILITY command to upstream"; + + line.len = s->tag.len + sizeof("AUTHENTICATE ") - 1 + + 1 + NGX_SIZE_T_LEN + 1 + 2; + line.data = ngx_pnalloc(c->pool, line.len); + if (line.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + + line.len = ngx_sprintf(line.data, "CAPABILITY" CRLF) + - line.data; + + s->mail_state = ngx_sieve_login_capabilities; + break; + + case ngx_sieve_login_capabilities: + s->connection->read->handler = ngx_mail_proxy_handler; + s->connection->write->handler = ngx_mail_proxy_handler; + rev->handler = ngx_mail_proxy_handler; + c->write->handler = ngx_mail_proxy_handler; + + pcf = ngx_mail_get_module_srv_conf(s, ngx_mail_proxy_module); + ngx_add_timer(s->connection->read, pcf->timeout); + ngx_del_timer(c->read); + + c->log->action = NULL; + ngx_log_error(NGX_LOG_INFO, c->log, 0, "client logged in"); + + ngx_mail_proxy_handler(s->connection->write); + + return; + + default: +#if (NGX_SUPPRESS_WARN) + ngx_str_null(&line); +#endif + break; + } + + if (c->send(c, line.data, line.len) < (ssize_t) line.len) { + /* + * we treat the incomplete sending as NGX_ERROR + * because it is very strange here + */ + ngx_mail_proxy_internal_server_error(s); + return; + } + + s->proxy->buffer->pos = s->proxy->buffer->start; + s->proxy->buffer->last = s->proxy->buffer->start; +} + + +static void ngx_mail_proxy_smtp_handler(ngx_event_t *rev) { u_char *p; @@ -793,6 +937,42 @@ break; + case NGX_MAIL_SIEVE_PROTOCOL: + if (p[0] == '"') { + m = b->last - (sizeof(CRLF "OK" CRLF) - 1); + + while (m > p) { + if (m[0] == CR && m[1] == LF) { + break; + } + m--; + } + + if (m <= p || m[0] == '"') { + return NGX_AGAIN; + } + p = m + 2; + } + + switch (state) { + + case ngx_sieve_start: + case ngx_sieve_login: + ngx_log_error(NGX_LOG_INFO, s->connection->log, 0, "1 upstream sent: \"%s\"", p); + if (p[0] == 'O' && p[1] == 'K') { + return NGX_OK; + } + break; + case ngx_sieve_login_capabilities: + ngx_log_error(NGX_LOG_INFO, s->connection->log, 0, "2 upstream sent: \"%s\"", p); + if (p[0] == 'O' && p[1] == 'K') { + return NGX_OK; + } + break; + } + + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ if (p[3] == '-') { diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_smtp_handler.c --- a/src/mail/ngx_mail_smtp_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_smtp_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -693,7 +693,7 @@ } } - if (ngx_mail_auth_cram_md5_salt(s, c, "334 ", 4) == NGX_OK) { + if (ngx_mail_auth_cram_md5_salt(s, c, "334 ", 4, 0) == NGX_OK) { s->mail_state = ngx_smtp_auth_cram_md5; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/misc/ngx_cpp_test_module.cpp --- a/src/misc/ngx_cpp_test_module.cpp Fri Mar 13 02:12:10 2020 +0300 +++ b/src/misc/ngx_cpp_test_module.cpp Wed Apr 08 20:17:11 2020 +0200 @@ -13,6 +13,7 @@ #include <ngx_mail.h> #include <ngx_mail_pop3_module.h> #include <ngx_mail_imap_module.h> + #include <ngx_mail_sieve_module.h> #include <ngx_mail_smtp_module.h> }

10 months

Re: PCRE2 support?

by Maxim Dounin

Hello! On Tue, Sep 18, 2018 at 08:12:20AM -0400, Thomas Ward wrote: > Downstream in Ubuntu, it has been proposed to demote pcre3 and > use pcre2 instead as it is newer. > https://trac.nginx.org/nginx/ticket/720 shows it was marked 4 > years ago that NGINX does not support pcre2. Are there any > plans to use pcre2 instead of pcre3? There are no immediate plans. When we last checked, there were no problems with PCRE, but PCRE2 wasn't available in most distributions we support, making the switch mostly meaningless. Also, it looks like PCRE2 is still not supported even by Exim, which is the parent project of PCRE and PCRE2: https://bugs.exim.org/show_bug.cgi?id=1878 As such, adding PCRE2 support to nginx looks premature. -- Maxim Dounin http://mdounin.ru/

10 months

Adding a fd that is not obtained through accept to the list the active connections

by Ottavio Campana

Hello, I want to write a module for an nginx that runs on a device with a private IP address and behind NAT, connects to a remote server and adds the newly created connection to the list of connections handled by nginx. At this point the remote server will invert the connection and start making requests. I tried studying the documentation on nginx.org, but I am not able to get an idea about how to achieve this. Can you please give me a suggestion about how to do it? Thank you, Ottavio -- Non c'è più forza nella normalità, c'è solo monotonia

11 months, 2 weeks

[nginx] release-1.21.2 tag

by Maxim Dounin

details: https://hg.nginx.org/nginx/rev/b2d1a602b241 branches: changeset: 7919:b2d1a602b241 user: Maxim Dounin <mdounin(a)mdounin.ru> date: Tue Aug 31 18:13:47 2021 +0300 description: release-1.21.2 tag diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r bfbc52374adc -r b2d1a602b241 .hgtags --- a/.hgtags Tue Aug 31 18:13:46 2021 +0300 +++ b/.hgtags Tue Aug 31 18:13:47 2021 +0300 @@ -462,3 +462,4 @@ da571b8eaf8f30f36c43b3c9b25e01e31f47149c ffcbb9980ee2bad27b4d7b1cd680b14ff47b29aa release-1.19.10 df34dcc9ac072ffd0945e5a1f3eb7987e8275375 release-1.21.0 a68ac0677f8553b1f84d357bc9da114731ab5f47 release-1.21.1 +bfbc52374adcbf2f9060afd62de940f6fab3bba5 release-1.21.2

11 months, 2 weeks

[nginx] nginx-1.21.2-RELEASE

by Maxim Dounin

details: https://hg.nginx.org/nginx/rev/bfbc52374adc branches: changeset: 7918:bfbc52374adc user: Maxim Dounin <mdounin(a)mdounin.ru> date: Tue Aug 31 18:13:46 2021 +0300 description: nginx-1.21.2-RELEASE diffstat: docs/xml/nginx/changes.xml | 107 +++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 107 insertions(+), 0 deletions(-) diffs (117 lines): diff -r f0ab1db646d5 -r bfbc52374adc docs/xml/nginx/changes.xml --- a/docs/xml/nginx/changes.xml Tue Aug 31 17:54:54 2021 +0300 +++ b/docs/xml/nginx/changes.xml Tue Aug 31 18:13:46 2021 +0300 @@ -5,6 +5,113 @@ <change_log title="nginx"> +<changes ver="1.21.2" date="2021-08-31"> + +<change type="change"> +<para lang="ru"> +теперь nginx возвращает ошибку, +если в запросе по протоколу HTTP/1.0 присутствует +строка заголовка "Transfer-Encoding". +</para> +<para lang="en"> +now nginx rejects HTTP/1.0 requests +with the "Transfer-Encoding" header line. +</para> +</change> + +<change type="change"> +<para lang="ru"> +экспортные шифры больше не поддерживаются. +</para> +<para lang="en"> +export ciphers are no longer supported. +</para> +</change> + +<change type="feature"> +<para lang="ru"> +совместимость с OpenSSL 3.0. +</para> +<para lang="en"> +OpenSSL 3.0 compatibility. +</para> +</change> + +<change type="feature"> +<para lang="ru"> +теперь серверу аутентификации почтового прокси-сервера +передаются строки заголовка "Auth-SSL-Protocol" и "Auth-SSL-Cipher".<br/> +Спасибо Rob Mueller. +</para> +<para lang="en"> +the "Auth-SSL-Protocol" and "Auth-SSL-Cipher" header lines +are now passed to the mail proxy authentication server.<br/> +Thanks to Rob Mueller. +</para> +</change> + +<change type="feature"> +<para lang="ru"> +API для обработки тела запроса +теперь позволяет буферизировать обрабатываемые данные. +</para> +<para lang="en"> +request body filters API +now permits buffering of the data being processed. +</para> +</change> + +<change type="bugfix"> +<para lang="ru"> +SSL-соединения к бэкендам в модуле stream +могли зависать после SSL handshake. +</para> +<para lang="en"> +backend SSL connections in the stream module +might hang after an SSL handshake. +</para> +</change> + +<change type="bugfix"> +<para lang="ru"> +уровень безопасности, доступный в OpenSSL 1.1.0 и новее, +не учитывался при загрузке сертификатов сервера, +если был задан через "@SECLEVEL=N" в директиве ssl_ciphers. +</para> +<para lang="en"> +the security level, which is available in OpenSSL 1.1.0 or newer, +did not affect loading of the server certificates +when set with "@SECLEVEL=N" in the "ssl_ciphers" directive. +</para> +</change> + +<change type="bugfix"> +<para lang="ru"> +SSL-соединения с gRPC-бэкендами могли зависать, +если использовались методы select, poll или /dev/poll. +</para> +<para lang="en"> +SSL connections with gRPC backends might hang +if select, poll, or /dev/poll methods were used. +</para> +</change> + +<change type="bugfix"> +<para lang="ru"> +при использовании HTTP/2 +тело запроса всегда записывалось на диск, +если в запросе не было строки заголовка "Content-Length". +</para> +<para lang="en"> +when using HTTP/2 +client request body was always written to disk +if the "Content-Length" header line was not present in the request. +</para> +</change> + +</changes> + + <changes ver="1.21.1" date="2021-07-06"> <change type="change">

11 months, 2 weeks

[nginx] Updated OpenSSL used for win32 builds.

by Maxim Dounin

details: https://hg.nginx.org/nginx/rev/f0ab1db646d5 branches: changeset: 7917:f0ab1db646d5 user: Maxim Dounin <mdounin(a)mdounin.ru> date: Tue Aug 31 17:54:54 2021 +0300 description: Updated OpenSSL used for win32 builds. diffstat: misc/GNUmakefile | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 29795b697e14 -r f0ab1db646d5 misc/GNUmakefile --- a/misc/GNUmakefile Tue Aug 31 16:44:13 2021 +0300 +++ b/misc/GNUmakefile Tue Aug 31 17:54:54 2021 +0300 @@ -6,7 +6,7 @@ TEMP = tmp CC = cl OBJS = objs.msvc8 -OPENSSL = openssl-1.1.1k +OPENSSL = openssl-1.1.1l ZLIB = zlib-1.2.11 PCRE = pcre-8.44

11 months, 2 weeks

[njs] Added tag 0.6.2 for changeset dfba7f61745c

by Dmitry Volyntsev

details: https://hg.nginx.org/njs/rev/65cd982d134f branches: changeset: 1696:65cd982d134f user: Dmitry Volyntsev <xeioex(a)nginx.com> date: Tue Aug 31 14:04:23 2021 +0000 description: Added tag 0.6.2 for changeset dfba7f61745c diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r dfba7f61745c -r 65cd982d134f .hgtags --- a/.hgtags Tue Aug 31 13:38:39 2021 +0000 +++ b/.hgtags Tue Aug 31 14:04:23 2021 +0000 @@ -44,3 +44,4 @@ e5de01378b1a8ab0a94dd3a8c4c6bb7a235f4b9c 282b9412976ceee31eb12876f1499fe975e6f08c 0.5.3 742ebceef2b5d15febc093172fe6174e427b26c8 0.6.0 4adbe67b292af2adc0a6fde4ec6cb95dbba9470a 0.6.1 +dfba7f61745c7454ffdd55303a793206d0a9a84a 0.6.2

11 months, 2 weeks

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

nginx-devel August 2021