nginx-devel October 2020

nginx-devel@nginx.org

13 participants
46 discussions

[PATCH] Added support for proxying managesieve protocol

by Sander Hoentjen

# HG changeset patch # User Sander Hoentjen <shoentjen(a)antagonist.nl> # Date 1586369831 -7200 # Wed Apr 08 20:17:11 2020 +0200 # Node ID f1dffaf619688aaab90caf31781ebe27c3f79598 # Parent 0cb942c1c1aa98118076e72e0b89940e85e6291c Added support for proxying managesieve protocol Sieve is a widely used protocol to implement server-side filtering. Some servers and clients that support this are listed at <http://sieve.info/>. This commit is tested with Dovecot as the server and Roundcube as the client, but I don't see a readon why it not would work with others. It is a nice addition to nginx because it adds the possibility to add server side filtering to setups that use IMAP behind an nginx proxy. Fixes https://trac.nginx.org/nginx/ticket/1697 diff -r 0cb942c1c1aa -r f1dffaf61968 auto/modules --- a/auto/modules Fri Mar 13 02:12:10 2020 +0300 +++ b/auto/modules Wed Apr 08 20:17:11 2020 +0200 @@ -965,6 +965,15 @@ . auto/module fi + if [ $MAIL_SIEVE = YES ]; then + ngx_module_name=ngx_mail_sieve_module + ngx_module_deps=src/mail/ngx_mail_sieve_module.h + ngx_module_srcs="src/mail/ngx_mail_sieve_module.c \ + src/mail/ngx_mail_sieve_handler.c" + + . auto/module + fi + if [ $MAIL_SMTP = YES ]; then ngx_module_name=ngx_mail_smtp_module ngx_module_deps=src/mail/ngx_mail_smtp_module.h diff -r 0cb942c1c1aa -r f1dffaf61968 auto/options --- a/auto/options Fri Mar 13 02:12:10 2020 +0300 +++ b/auto/options Wed Apr 08 20:17:11 2020 +0200 @@ -112,6 +112,7 @@ MAIL_SSL=NO MAIL_POP3=YES MAIL_IMAP=YES +MAIL_SIEVE=YES MAIL_SMTP=YES STREAM=NO @@ -305,6 +306,8 @@ ;; --without-mail_pop3_module) MAIL_POP3=NO ;; --without-mail_imap_module) MAIL_IMAP=NO ;; + --without-mail_sieve_module) + MAIL_SIEVE=NO ;; --without-mail_smtp_module) MAIL_SMTP=NO ;; --with-stream) STREAM=YES ;; @@ -517,11 +520,12 @@ --without-http disable HTTP server --without-http-cache disable HTTP cache - --with-mail enable POP3/IMAP4/SMTP proxy module - --with-mail=dynamic enable dynamic POP3/IMAP4/SMTP proxy module + --with-mail enable POP3/IMAP4/SIEVE/SMTP proxy module + --with-mail=dynamic enable dynamic POP3/IMAP4/SIEVE/SMTP proxy module --with-mail_ssl_module enable ngx_mail_ssl_module --without-mail_pop3_module disable ngx_mail_pop3_module --without-mail_imap_module disable ngx_mail_imap_module + --without-mail_sieve_module disable ngx_mail_sieve_module --without-mail_smtp_module disable ngx_mail_smtp_module --with-stream enable TCP/UDP proxy module diff -r 0cb942c1c1aa -r f1dffaf61968 contrib/vim/syntax/nginx.vim --- a/contrib/vim/syntax/nginx.vim Fri Mar 13 02:12:10 2020 +0300 +++ b/contrib/vim/syntax/nginx.vim Wed Apr 08 20:17:11 2020 +0200 @@ -571,6 +571,9 @@ syn keyword ngxDirective contained session_log_format syn keyword ngxDirective contained session_log_zone syn keyword ngxDirective contained set_real_ip_from +syn keyword ngxDirective contained sieve_auth +syn keyword ngxDirective contained sieve_capabilities +syn keyword ngxDirective contained sieve_client_buffer syn keyword ngxDirective contained slice syn keyword ngxDirective contained smtp_auth syn keyword ngxDirective contained smtp_capabilities diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail.h --- a/src/mail/ngx_mail.h Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail.h Wed Apr 08 20:17:11 2020 +0200 @@ -102,6 +102,7 @@ #define NGX_MAIL_POP3_PROTOCOL 0 #define NGX_MAIL_IMAP_PROTOCOL 1 #define NGX_MAIL_SMTP_PROTOCOL 2 +#define NGX_MAIL_SIEVE_PROTOCOL 3 typedef struct ngx_mail_protocol_s ngx_mail_protocol_t; @@ -154,6 +155,21 @@ typedef enum { + ngx_sieve_start = 0, + ngx_sieve_starttls, + ngx_sieve_auth_login_username, + ngx_sieve_auth_login_password, + ngx_sieve_auth_plain, + ngx_sieve_auth_cram_md5, + ngx_sieve_auth_external, + ngx_sieve_login, + ngx_sieve_login_capabilities, + ngx_sieve_user, + ngx_sieve_passwd +} ngx_sieve_state_e; + + +typedef enum { ngx_smtp_start = 0, ngx_smtp_auth_login_username, ngx_smtp_auth_login_password, @@ -227,7 +243,7 @@ ngx_uint_t login_attempt; - /* used to parse POP3/IMAP/SMTP command */ + /* used to parse POP3/IMAP/SIEVE/SMTP command */ ngx_uint_t state; u_char *cmd_start; @@ -271,6 +287,14 @@ #define NGX_IMAP_AUTHENTICATE 7 +#define NGX_SIEVE_LOGOUT 1 +#define NGX_SIEVE_CAPABILITY 2 +#define NGX_SIEVE_NOOP 3 +#define NGX_SIEVE_STARTTLS 4 +#define NGX_SIEVE_NEXT 5 +#define NGX_SIEVE_AUTHENTICATE 6 + + #define NGX_SMTP_HELO 1 #define NGX_SMTP_EHLO 2 #define NGX_SMTP_AUTH 3 @@ -383,7 +407,7 @@ ngx_int_t ngx_mail_auth_login_password(ngx_mail_session_t *s, ngx_connection_t *c); ngx_int_t ngx_mail_auth_cram_md5_salt(ngx_mail_session_t *s, - ngx_connection_t *c, char *prefix, size_t len); + ngx_connection_t *c, char *prefix, size_t len, ngx_uint_t quote); ngx_int_t ngx_mail_auth_cram_md5(ngx_mail_session_t *s, ngx_connection_t *c); ngx_int_t ngx_mail_auth_external(ngx_mail_session_t *s, ngx_connection_t *c, ngx_uint_t n); diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_auth_http_module.c --- a/src/mail/ngx_mail_auth_http_module.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_auth_http_module.c Wed Apr 08 20:17:11 2020 +0200 @@ -533,6 +533,11 @@ + sizeof(CRLF) - 1; break; + case NGX_MAIL_SIEVE_PROTOCOL: + size = sizeof("BYE \"\"") - 1 + len + + sizeof(CRLF) - 1; + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ ctx->err = ctx->errmsg; continue; @@ -559,11 +564,27 @@ *p++ = 'N'; *p++ = 'O'; *p++ = ' '; break; + case NGX_MAIL_SIEVE_PROTOCOL: + p = ngx_cpymem(p, s->tag.data, s->tag.len); + *p++ = 'B'; *p++ = 'Y'; *p++ = 'E'; *p++ = ' '; *p++ = '"'; + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ break; } p = ngx_cpymem(p, ctx->header_start, len); + + switch (s->protocol) { + + case NGX_MAIL_SIEVE_PROTOCOL: + *p++ = '"'; + break; + + default: + break; + } + *p++ = CR; *p++ = LF; ctx->err.len = p - ctx->err.data; diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_handler.c --- a/src/mail/ngx_mail_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -520,28 +520,39 @@ ngx_int_t ngx_mail_auth_cram_md5_salt(ngx_mail_session_t *s, ngx_connection_t *c, - char *prefix, size_t len) + char *prefix, size_t len, ngx_uint_t quote) { u_char *p; ngx_str_t salt; - ngx_uint_t n; + ngx_uint_t n = 0; + if (quote) { + len += 2; + } p = ngx_pnalloc(c->pool, len + ngx_base64_encoded_length(s->salt.len) + 2); if (p == NULL) { return NGX_ERROR; } - salt.data = ngx_cpymem(p, prefix, len); + if (quote) { + len -= 2; + salt.data = ngx_cpymem(p, "\"", 1); + n++; + } + salt.data = ngx_cpymem(p+ n, prefix, len); s->salt.len -= 2; ngx_encode_base64(&salt, &s->salt); s->salt.len += 2; - n = len + salt.len; + n += len + salt.len; + if (quote) { + p[n++] = '"'; + } p[n++] = CR; p[n++] = LF; s->out.len = n; - s->out.data = p; + s->out.data = p--; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_imap_handler.c --- a/src/mail/ngx_mail_imap_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_imap_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -397,7 +397,7 @@ } } - if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2) == NGX_OK) { + if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2, 0) == NGX_OK) { s->mail_state = ngx_imap_auth_cram_md5; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_parse.c --- a/src/mail/ngx_mail_parse.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_parse.c Wed Apr 08 20:17:11 2020 +0200 @@ -11,6 +11,7 @@ #include <ngx_mail.h> #include <ngx_mail_pop3_module.h> #include <ngx_mail_imap_module.h> +#include <ngx_mail_sieve_module.h> #include <ngx_mail_smtp_module.h> @@ -620,6 +621,364 @@ ngx_int_t +ngx_mail_sieve_parse_command(ngx_mail_session_t *s) +{ + u_char ch, *p, *c; + ngx_str_t *arg; + enum { + sw_start = 0, + sw_spaces_before_argument, + sw_argument, + sw_backslash, + sw_literal, + sw_no_sync_literal_argument, + sw_start_literal_argument, + sw_literal_argument, + sw_end_literal_argument, + sw_almost_done + } state; + + state = s->state; + + for (p = s->buffer->pos; p < s->buffer->last; p++) { + ch = *p; + + switch (state) { + + /* SIEVE command */ + case sw_start: + if (ch == ' ' || ch == CR || ch == LF) { + + c = s->buffer->start; + + switch (p - c) { + + case 4: + if ((c[0] == 'N' || c[0] == 'n') + && (c[1] == 'O'|| c[1] == 'o') + && (c[2] == 'O'|| c[2] == 'o') + && (c[3] == 'P'|| c[3] == 'p')) + { + s->command = NGX_SIEVE_NOOP; + + } else { + goto invalid; + } + break; + + case 6: + if ((c[0] == 'L'|| c[0] == 'l') + && (c[1] == 'O'|| c[1] == 'o') + && (c[2] == 'G'|| c[2] == 'g') + && (c[3] == 'O'|| c[3] == 'o') + && (c[4] == 'U'|| c[4] == 'u') + && (c[5] == 'T'|| c[5] == 't')) + { + s->command = NGX_SIEVE_LOGOUT; + + } else { + goto invalid; + } + break; + +#if (NGX_MAIL_SSL) + case 8: + if ((c[0] == 'S'|| c[0] == 's') + && (c[1] == 'T'|| c[1] == 't') + && (c[2] == 'A'|| c[2] == 'a') + && (c[3] == 'R'|| c[3] == 'r') + && (c[4] == 'T'|| c[4] == 't') + && (c[5] == 'T'|| c[5] == 't') + && (c[6] == 'L'|| c[6] == 'l') + && (c[7] == 'S'|| c[7] == 's')) + { + s->command = NGX_SIEVE_STARTTLS; + + } else { + goto invalid; + } + break; +#endif + + case 10: + if ((c[0] == 'C'|| c[0] == 'c') + && (c[1] == 'A'|| c[1] == 'a') + && (c[2] == 'P'|| c[2] == 'p') + && (c[3] == 'A'|| c[3] == 'a') + && (c[4] == 'B'|| c[4] == 'b') + && (c[5] == 'I'|| c[5] == 'i') + && (c[6] == 'L'|| c[6] == 'l') + && (c[7] == 'I'|| c[7] == 'i') + && (c[8] == 'T'|| c[8] == 't') + && (c[9] == 'Y'|| c[9] == 'y')) + { + s->command = NGX_SIEVE_CAPABILITY; + + } else { + goto invalid; + } + break; + + case 12: + if ((c[0] == 'A'|| c[0] == 'a') + && (c[1] == 'U'|| c[1] == 'u') + && (c[2] == 'T'|| c[2] == 't') + && (c[3] == 'H'|| c[3] == 'h') + && (c[4] == 'E'|| c[4] == 'e') + && (c[5] == 'N'|| c[5] == 'n') + && (c[6] == 'T'|| c[6] == 't') + && (c[7] == 'I'|| c[7] == 'i') + && (c[8] == 'C'|| c[8] == 'c') + && (c[9] == 'A'|| c[9] == 'a') + && (c[10] == 'T'|| c[10] == 't') + && (c[11] == 'E'|| c[11] == 'e')) + { + s->command = NGX_SIEVE_AUTHENTICATE; + + } else { + goto invalid; + } + break; + + default: + goto invalid; + } + + switch (ch) { + case ' ': + state = sw_spaces_before_argument; + break; + case CR: + state = sw_almost_done; + break; + case LF: + goto done; + } + break; + } + + if ((ch < 'A' || ch > 'Z') && (ch < 'a' || ch > 'z')) { + goto invalid; + } + + break; + + case sw_spaces_before_argument: + switch (ch) { + case ' ': + break; + case CR: + state = sw_almost_done; + s->arg_end = p; + break; + case LF: + s->arg_end = p; + goto done; + case '"': + if (s->args.nelts <= 2) { + s->quoted = 1; + s->arg_start = p + 1; + state = sw_argument; + break; + } + goto invalid; + case '{': + if (s->args.nelts <= 2) { + state = sw_literal; + break; + } + goto invalid; + default: + if (s->args.nelts <= 2) { + s->arg_start = p; + state = sw_argument; + break; + } + goto invalid; + } + break; + + case sw_argument: + if (ch == ' ' && s->quoted) { + break; + } + + switch (ch) { + case '"': + if (!s->quoted) { + break; + } + s->quoted = 0; + /* fall through */ + case ' ': + case CR: + case LF: + arg = ngx_array_push(&s->args); + if (arg == NULL) { + return NGX_ERROR; + } + arg->len = p - s->arg_start; + arg->data = s->arg_start; + s->arg_start = NULL; + + switch (ch) { + case '"': + case ' ': + state = sw_spaces_before_argument; + break; + case CR: + state = sw_almost_done; + break; + case LF: + goto done; + } + break; + case '\\': + if (s->quoted) { + s->backslash = 1; + state = sw_backslash; + } + break; + } + break; + + case sw_backslash: + switch (ch) { + case CR: + case LF: + goto invalid; + default: + state = sw_argument; + } + break; + + case sw_literal: + if (ch >= '0' && ch <= '9') { + s->literal_len = s->literal_len * 10 + (ch - '0'); + break; + } + if (ch == '}') { + state = sw_start_literal_argument; + break; + } + if (ch == '+') { + state = sw_no_sync_literal_argument; + break; + } + goto invalid; + + case sw_no_sync_literal_argument: + if (ch == '}') { + s->no_sync_literal = 1; + state = sw_start_literal_argument; + break; + } + goto invalid; + + case sw_start_literal_argument: + switch (ch) { + case CR: + break; + case LF: + s->buffer->pos = p + 1; + s->arg_start = p + 1; + if (s->no_sync_literal == 0) { + s->state = sw_literal_argument; + return NGX_SIEVE_NEXT; + } + state = sw_literal_argument; + s->no_sync_literal = 0; + break; + default: + goto invalid; + } + break; + + case sw_literal_argument: + if (s->literal_len && --s->literal_len) { + break; + } + + arg = ngx_array_push(&s->args); + if (arg == NULL) { + return NGX_ERROR; + } + arg->len = p + 1 - s->arg_start; + arg->data = s->arg_start; + s->arg_start = NULL; + state = sw_end_literal_argument; + + break; + + case sw_end_literal_argument: + switch (ch) { + case '{': + if (s->args.nelts <= 2) { + state = sw_literal; + break; + } + goto invalid; + case CR: + state = sw_almost_done; + break; + case LF: + goto done; + default: + state = sw_spaces_before_argument; + break; + } + break; + + case sw_almost_done: + switch (ch) { + case LF: + goto done; + default: + goto invalid; + } + } + } + + s->buffer->pos = p; + s->state = state; + + return NGX_AGAIN; + +done: + + s->buffer->pos = p + 1; + + if (s->arg_start) { + arg = ngx_array_push(&s->args); + if (arg == NULL) { + return NGX_ERROR; + } + arg->len = s->arg_end - s->arg_start; + arg->data = s->arg_start; + + s->arg_start = NULL; + s->cmd_start = NULL; + s->quoted = 0; + s->no_sync_literal = 0; + s->literal_len = 0; + } + + s->state = (s->command != NGX_SIEVE_AUTHENTICATE) ? sw_start : sw_spaces_before_argument; + + return NGX_OK; + +invalid: + + s->state = sw_start; + s->quoted = 0; + s->no_sync_literal = 0; + s->literal_len = 0; + + return NGX_MAIL_PARSE_INVALID_COMMAND; +} + + +ngx_int_t ngx_mail_smtp_parse_command(ngx_mail_session_t *s) { u_char ch, *p, *c, c0, c1, c2, c3; diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_pop3_handler.c --- a/src/mail/ngx_mail_pop3_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_pop3_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -492,7 +492,7 @@ return NGX_MAIL_PARSE_INVALID_COMMAND; } - if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2) == NGX_OK) { + if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2, 0) == NGX_OK) { s->mail_state = ngx_pop3_auth_cram_md5; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_proxy_module.c --- a/src/mail/ngx_mail_proxy_module.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_proxy_module.c Wed Apr 08 20:17:11 2020 +0200 @@ -24,6 +24,7 @@ static void ngx_mail_proxy_block_read(ngx_event_t *rev); static void ngx_mail_proxy_pop3_handler(ngx_event_t *rev); static void ngx_mail_proxy_imap_handler(ngx_event_t *rev); +static void ngx_mail_proxy_sieve_handler(ngx_event_t *rev); static void ngx_mail_proxy_smtp_handler(ngx_event_t *rev); static void ngx_mail_proxy_dummy_handler(ngx_event_t *ev); static ngx_int_t ngx_mail_proxy_read_response(ngx_mail_session_t *s, @@ -173,6 +174,11 @@ s->mail_state = ngx_imap_start; break; + case NGX_MAIL_SIEVE_PROTOCOL: + p->upstream.connection->read->handler = ngx_mail_proxy_sieve_handler; + s->mail_state = ngx_sieve_start; + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ p->upstream.connection->read->handler = ngx_mail_proxy_smtp_handler; s->mail_state = ngx_smtp_start; @@ -446,6 +452,144 @@ static void +ngx_mail_proxy_sieve_handler(ngx_event_t *rev) +{ + u_char *p; + ngx_int_t rc; + ngx_str_t line; + ngx_str_t userpass; + ngx_str_t b64_userpass; + ngx_connection_t *c; + ngx_mail_session_t *s; + ngx_mail_proxy_conf_t *pcf; + + ngx_log_debug0(NGX_LOG_DEBUG_MAIL, rev->log, 0, + "mail proxy sieve auth handler"); + + c = rev->data; + s = c->data; + + if (rev->timedout) { + ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, + "upstream timed out"); + c->timedout = 1; + ngx_mail_proxy_internal_server_error(s); + return; + } + + rc = ngx_mail_proxy_read_response(s, s->mail_state); + + if (rc == NGX_AGAIN) { + return; + } + + if (rc == NGX_ERROR) { + ngx_mail_proxy_upstream_error(s); + return; + } + + switch (s->mail_state) { + + case ngx_sieve_start: + ngx_log_debug0(NGX_LOG_DEBUG_MAIL, rev->log, 0, + "mail proxy send login"); + + s->connection->log->action = "sending LOGIN command to upstream"; + + line.len = s->tag.len + sizeof("AUTHENTICATE ") - 1 + + 1 + NGX_SIZE_T_LEN + 1 + 2; + line.data = ngx_pnalloc(c->pool, line.len); + if (line.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + + userpass.len = s->login.len + s->passwd.len + 2; + userpass.data = ngx_pnalloc(c->pool, userpass.len); + if (userpass.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + b64_userpass.len = ngx_base64_encoded_length(userpass.len) + 2; + b64_userpass.data = ngx_pnalloc(c->pool, b64_userpass.len); + if (b64_userpass.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + + p = userpass.data; + *p++ = '\0'; + p = ngx_cpymem(p, s->login.data, s->login.len); + *p++ = '\0'; + p = ngx_cpymem(p, s->passwd.data, s->passwd.len); + + ngx_encode_base64(&b64_userpass, &userpass); + line.len = ngx_sprintf(line.data, "AUTHENTICATE \"PLAIN\" \"%V\"" CRLF, + &b64_userpass) + - line.data; + + s->mail_state = ngx_sieve_login; + break; + + case ngx_sieve_login: + ngx_log_debug0(NGX_LOG_DEBUG_MAIL, rev->log, 0, + "mail proxy send capabilities"); + + s->connection->log->action = "sending CAPABILITY command to upstream"; + + line.len = s->tag.len + sizeof("AUTHENTICATE ") - 1 + + 1 + NGX_SIZE_T_LEN + 1 + 2; + line.data = ngx_pnalloc(c->pool, line.len); + if (line.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + + line.len = ngx_sprintf(line.data, "CAPABILITY" CRLF) + - line.data; + + s->mail_state = ngx_sieve_login_capabilities; + break; + + case ngx_sieve_login_capabilities: + s->connection->read->handler = ngx_mail_proxy_handler; + s->connection->write->handler = ngx_mail_proxy_handler; + rev->handler = ngx_mail_proxy_handler; + c->write->handler = ngx_mail_proxy_handler; + + pcf = ngx_mail_get_module_srv_conf(s, ngx_mail_proxy_module); + ngx_add_timer(s->connection->read, pcf->timeout); + ngx_del_timer(c->read); + + c->log->action = NULL; + ngx_log_error(NGX_LOG_INFO, c->log, 0, "client logged in"); + + ngx_mail_proxy_handler(s->connection->write); + + return; + + default: +#if (NGX_SUPPRESS_WARN) + ngx_str_null(&line); +#endif + break; + } + + if (c->send(c, line.data, line.len) < (ssize_t) line.len) { + /* + * we treat the incomplete sending as NGX_ERROR + * because it is very strange here + */ + ngx_mail_proxy_internal_server_error(s); + return; + } + + s->proxy->buffer->pos = s->proxy->buffer->start; + s->proxy->buffer->last = s->proxy->buffer->start; +} + + +static void ngx_mail_proxy_smtp_handler(ngx_event_t *rev) { u_char *p; @@ -793,6 +937,42 @@ break; + case NGX_MAIL_SIEVE_PROTOCOL: + if (p[0] == '"') { + m = b->last - (sizeof(CRLF "OK" CRLF) - 1); + + while (m > p) { + if (m[0] == CR && m[1] == LF) { + break; + } + m--; + } + + if (m <= p || m[0] == '"') { + return NGX_AGAIN; + } + p = m + 2; + } + + switch (state) { + + case ngx_sieve_start: + case ngx_sieve_login: + ngx_log_error(NGX_LOG_INFO, s->connection->log, 0, "1 upstream sent: \"%s\"", p); + if (p[0] == 'O' && p[1] == 'K') { + return NGX_OK; + } + break; + case ngx_sieve_login_capabilities: + ngx_log_error(NGX_LOG_INFO, s->connection->log, 0, "2 upstream sent: \"%s\"", p); + if (p[0] == 'O' && p[1] == 'K') { + return NGX_OK; + } + break; + } + + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ if (p[3] == '-') { diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_smtp_handler.c --- a/src/mail/ngx_mail_smtp_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_smtp_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -693,7 +693,7 @@ } } - if (ngx_mail_auth_cram_md5_salt(s, c, "334 ", 4) == NGX_OK) { + if (ngx_mail_auth_cram_md5_salt(s, c, "334 ", 4, 0) == NGX_OK) { s->mail_state = ngx_smtp_auth_cram_md5; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/misc/ngx_cpp_test_module.cpp --- a/src/misc/ngx_cpp_test_module.cpp Fri Mar 13 02:12:10 2020 +0300 +++ b/src/misc/ngx_cpp_test_module.cpp Wed Apr 08 20:17:11 2020 +0200 @@ -13,6 +13,7 @@ #include <ngx_mail.h> #include <ngx_mail_pop3_module.h> #include <ngx_mail_imap_module.h> + #include <ngx_mail_sieve_module.h> #include <ngx_mail_smtp_module.h> }

1 year, 1 month

Re: PCRE2 support?

by Maxim Dounin

Hello! On Tue, Sep 18, 2018 at 08:12:20AM -0400, Thomas Ward wrote: > Downstream in Ubuntu, it has been proposed to demote pcre3 and > use pcre2 instead as it is newer. > https://trac.nginx.org/nginx/ticket/720 shows it was marked 4 > years ago that NGINX does not support pcre2. Are there any > plans to use pcre2 instead of pcre3? There are no immediate plans. When we last checked, there were no problems with PCRE, but PCRE2 wasn't available in most distributions we support, making the switch mostly meaningless. Also, it looks like PCRE2 is still not supported even by Exim, which is the parent project of PCRE and PCRE2: https://bugs.exim.org/show_bug.cgi?id=1878 As such, adding PCRE2 support to nginx looks premature. -- Maxim Dounin http://mdounin.ru/

1 year, 1 month

cache: move open to thread pool

by Ka-Hing Cheung

commit 0bcfefa040abdff674047c15701d237ff16a5f2b Author: Ka-Hing Cheung <kahing(a)cloudflare.com> Date: Fri Aug 3 13:37:58 2018 -0700 move open to thread pool At cloudflare we found that open() can block a long time, especially at p99 and p999. Moving it to thread pool improved overall p99 by 6x or so during peak time. This has the side effect of disabling nginx's open_file_cache when "aio threads" is turned on. For us we found that to have no impact (probably because we have way too many files for open file cache to be effective anyway). thread pool does increase CPU usage somewhat but we have not measured difference in CPU usage for stock "aio threads" compared to after this patch. Only the cache hit open() is moved to thread pool. We wrote more about it here: https://blog.cloudflare.com/how-we-scaled-nginx-and-saved-the-world-54-year… diff --git src/core/ngx_open_file_cache.c src/core/ngx_open_file_cache.c index b23ee78d..9177ebfd 100644 --- src/core/ngx_open_file_cache.c +++ src/core/ngx_open_file_cache.c @@ -35,8 +35,6 @@ static ngx_fd_t ngx_open_file_wrapper(ngx_str_t *name, ngx_int_t access, ngx_log_t *log); static ngx_int_t ngx_file_info_wrapper(ngx_str_t *name, ngx_open_file_info_t *of, ngx_file_info_t *fi, ngx_log_t *log); -static ngx_int_t ngx_open_and_stat_file(ngx_str_t *name, - ngx_open_file_info_t *of, ngx_log_t *log); static void ngx_open_file_add_event(ngx_open_file_cache_t *cache, ngx_cached_open_file_t *file, ngx_open_file_info_t *of, ngx_log_t *log); static void ngx_open_file_cleanup(void *data); @@ -836,7 +834,7 @@ ngx_file_info_wrapper(ngx_str_t *name, ngx_open_file_info_t *of, } -static ngx_int_t +ngx_int_t ngx_open_and_stat_file(ngx_str_t *name, ngx_open_file_info_t *of, ngx_log_t *log) { diff --git src/core/ngx_open_file_cache.h src/core/ngx_open_file_cache.h index d119c129..94b4d552 100644 --- src/core/ngx_open_file_cache.h +++ src/core/ngx_open_file_cache.h @@ -124,6 +124,8 @@ ngx_open_file_cache_t *ngx_open_file_cache_init(ngx_pool_t *pool, ngx_uint_t max, time_t inactive); ngx_int_t ngx_open_cached_file(ngx_open_file_cache_t *cache, ngx_str_t *name, ngx_open_file_info_t *of, ngx_pool_t *pool); +ngx_int_t ngx_open_and_stat_file(ngx_str_t *name, + ngx_open_file_info_t *of, ngx_log_t *log); #endif /* _NGX_OPEN_FILE_CACHE_H_INCLUDED_ */ diff --git src/http/ngx_http_cache.h src/http/ngx_http_cache.h index f9e96640..2910f6a1 100644 --- src/http/ngx_http_cache.h +++ src/http/ngx_http_cache.h @@ -114,6 +114,7 @@ struct ngx_http_cache_s { unsigned exists:1; unsigned temp_file:1; unsigned purged:1; + unsigned opening:1; unsigned reading:1; unsigned secondary:1; unsigned background:1; diff --git src/http/ngx_http_file_cache.c src/http/ngx_http_file_cache.c index 56866fa4..9cb110bf 100644 --- src/http/ngx_http_file_cache.c +++ src/http/ngx_http_file_cache.c @@ -18,6 +18,10 @@ static void ngx_http_file_cache_lock_wait(ngx_http_request_t *r, ngx_http_cache_t *c); static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c); +static ngx_int_t ngx_http_file_cache_aio_open(ngx_http_request_t *r, + ngx_http_cache_t *c, ngx_int_t *rv); +static ngx_int_t ngx_http_file_cache_do_aio_open(ngx_http_request_t *r, + ngx_http_cache_t *c, ngx_open_file_info_t *of, ngx_int_t *rv); static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c); #if (NGX_HAVE_FILE_AIO) @@ -268,9 +272,7 @@ ngx_http_file_cache_open(ngx_http_request_t *r) ngx_uint_t test; ngx_http_cache_t *c; ngx_pool_cleanup_t *cln; - ngx_open_file_info_t of; ngx_http_file_cache_t *cache; - ngx_http_core_loc_conf_t *clcf; c = r->cache; @@ -278,6 +280,10 @@ ngx_http_file_cache_open(ngx_http_request_t *r) return NGX_AGAIN; } + if (c->opening) { + return ngx_http_file_cache_aio_open(r, c, &rv); + } + if (c->reading) { return ngx_http_file_cache_read(r, c); } @@ -343,23 +349,33 @@ ngx_http_file_cache_open(ngx_http_request_t *r) goto done; } - clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + return ngx_http_file_cache_aio_open(r, c, &rv); - ngx_memzero(&of, sizeof(ngx_open_file_info_t)); +done: - of.uniq = c->uniq; - of.valid = clcf->open_file_cache_valid; - of.min_uses = clcf->open_file_cache_min_uses; - of.events = clcf->open_file_cache_events; - of.directio = NGX_OPEN_FILE_DIRECTIO_OFF; - of.read_ahead = clcf->read_ahead; + if (rv == NGX_DECLINED) { + return ngx_http_file_cache_lock(r, c); + } - if (ngx_open_cached_file(clcf->open_file_cache, &c->file.name, &of, r->pool) - != NGX_OK) - { - switch (of.err) { + return rv; +} + +static ngx_int_t +ngx_http_file_cache_aio_open(ngx_http_request_t *r, ngx_http_cache_t *c, + ngx_int_t *rv) +{ + ngx_int_t rc; + ngx_open_file_info_t of; + ngx_http_file_cache_t *cache = c->file_cache; - case 0: + rc = ngx_http_file_cache_do_aio_open(r, c, &of, rv); + if (rc == NGX_AGAIN) { + return rc; + } + + if (rc != NGX_OK) { + switch (of.err) { + case NGX_OK: return NGX_ERROR; case NGX_ENOENT: @@ -391,14 +407,13 @@ ngx_http_file_cache_open(ngx_http_request_t *r) done: - if (rv == NGX_DECLINED) { + if (*rv == NGX_DECLINED) { return ngx_http_file_cache_lock(r, c); } - return rv; + return *rv; } - static ngx_int_t ngx_http_file_cache_lock(ngx_http_request_t *r, ngx_http_cache_t *c) { @@ -663,6 +678,127 @@ ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c) } +#if (NGX_THREADS) +typedef struct { + ngx_str_t name; + ngx_pool_t *pool; + ngx_open_file_info_t of; + ngx_int_t rv; +} ngx_thread_file_open_ctx_t; + +static void +ngx_http_file_cache_thread_open_handler(void *data, ngx_log_t *log) +{ + ngx_thread_file_open_ctx_t *ctx = data; + ngx_pool_t *pool = ctx->pool; + ngx_open_file_info_t *of = &ctx->of; + ngx_pool_cleanup_t *cln; + ngx_pool_cleanup_file_t *clnf; + ngx_int_t rc; + + ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "thread read handler"); + + cln = ngx_pool_cleanup_add(pool, sizeof(ngx_pool_cleanup_file_t)); + if (cln == NULL) { + ctx->of.err = NGX_ERROR; + return; + } + + of->fd = NGX_INVALID_FILE; + + rc = ngx_open_and_stat_file(&ctx->name, of, pool->log); + if (rc == NGX_OK && !of->is_dir) { + cln->handler = ngx_pool_cleanup_file; + clnf = cln->data; + + clnf->fd = of->fd; + clnf->name = ctx->name.data; + clnf->log = pool->log; + } +} + + +static ngx_int_t +ngx_http_file_cache_thread_open(ngx_http_cache_t *c, ngx_open_file_info_t *of, + ngx_int_t *rv, ngx_pool_t *pool) +{ + ngx_thread_task_t *task; + ngx_thread_file_open_ctx_t *ctx; + ngx_file_t *file = &c->file; + + task = file->thread_task; + + if (task == NULL) { + task = ngx_thread_task_alloc(pool, sizeof(ngx_thread_file_open_ctx_t)); + if (task == NULL) { + return NGX_ERROR; + } + + file->thread_task = task; + } + + ctx = task->ctx; + + if (task->event.complete) { + task->event.complete = 0; + + *of = ctx->of; + *rv = ctx->rv; + return of->err; + } + + task->handler = ngx_http_file_cache_thread_open_handler; + + ctx->pool = pool; + ctx->name = file->name; + ctx->of = *of; + ctx->rv = *rv; + + if (file->thread_handler(task, file) != NGX_OK) { + return NGX_ERROR; + } + + return NGX_AGAIN; +} +#endif + +static ngx_int_t +ngx_http_file_cache_do_aio_open(ngx_http_request_t *r, ngx_http_cache_t *c, + ngx_open_file_info_t *of, ngx_int_t *rv) +{ + ngx_http_core_loc_conf_t *clcf; + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + + ngx_memzero(of, sizeof(ngx_open_file_info_t)); + + of->uniq = c->uniq; + of->valid = clcf->open_file_cache_valid; + of->min_uses = clcf->open_file_cache_min_uses; + of->events = clcf->open_file_cache_events; + of->directio = NGX_OPEN_FILE_DIRECTIO_OFF; + of->read_ahead = clcf->read_ahead; + +#if (NGX_THREADS) + if (clcf->aio == NGX_HTTP_AIO_THREADS) { + ngx_int_t rc; + + c->file.thread_task = c->thread_task; + c->file.thread_handler = ngx_http_cache_thread_handler; + c->file.thread_ctx = r; + + rc = ngx_http_file_cache_thread_open(c, of, rv, r->pool); + + c->thread_task = c->file.thread_task; + c->opening = (rc == NGX_AGAIN); + + return rc; + } +#endif + + return ngx_open_cached_file(clcf->open_file_cache, &c->file.name, of, r->pool); +} + static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c) {

1 year, 4 months

[PATCH] Use BPF to distribute packet to different work thread.

by Liu, Qiao

# HG changeset patch # User Liu Qiao <qiao.liu(a)intel.com> # Date 1599735293 14400 # Thu Sep 10 06:54:53 2020 -0400 # Node ID f79d524a2cc0093c53490f947564e42371cf944f # Parent da5e3f5b16733167142b599b6af3ce9469a07d52 Use BPF to distribute packet to different work thread. Use Berkeley Packet Filter to get packet queue_mapping number, and use this queue_mapping number to distribute the packet to different work thread, this will improve CPU utilization and http latency. Author: Samudrala, Sridhar <sridhar.samudrala(a)intel.com> diff -r da5e3f5b1673 -r f79d524a2cc0 auto/os/linux --- a/auto/os/linux Wed Sep 02 23:13:36 2020 +0300 +++ b/auto/os/linux Thu Sep 10 06:54:53 2020 -0400 @@ -32,6 +32,10 @@ have=NGX_HAVE_POSIX_FADVISE . auto/nohave fi +if [ $version -lt 263680 ]; then + have=NGX_HAVE_REUSEPORT_CBPF . auto/nohave +fi + # epoll, EPOLLET version ngx_feature="epoll" diff -r da5e3f5b1673 -r f79d524a2cc0 auto/unix --- a/auto/unix Wed Sep 02 23:13:36 2020 +0300 +++ b/auto/unix Thu Sep 10 06:54:53 2020 -0400 @@ -331,6 +331,17 @@ ngx_feature_test="setsockopt(0, SOL_SOCKET, SO_REUSEPORT, NULL, 0)" . auto/feature +ngx_feature="SO_REUSEPORT_CBPF" +ngx_feature_name="NGX_HAVE_REUSEPORT_CBPF" +ngx_feature_run=no +ngx_feature_incs="#include <sys/socket.h> + #include <linux/filter.h> + #include <error.h>" +ngx_feature_path= +ngx_feature_libs= +ngx_feature_test="setsockopt(0, SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF, NULL, 0)" +. auto/feature + ngx_feature="SO_ACCEPTFILTER" ngx_feature_name="NGX_HAVE_DEFERRED_ACCEPT" diff -r da5e3f5b1673 -r f79d524a2cc0 src/core/ngx_connection.c --- a/src/core/ngx_connection.c Wed Sep 02 23:13:36 2020 +0300 +++ b/src/core/ngx_connection.c Thu Sep 10 06:54:53 2020 -0400 @@ -8,7 +8,10 @@ #include <ngx_config.h> #include <ngx_core.h> #include <ngx_event.h> - +#if (NGX_HAVE_REUSEPORT_CBPF) +#include <linux/filter.h> +#include <error.h> +#endif ngx_os_io_t ngx_io; @@ -708,6 +711,35 @@ return NGX_OK; } +#if(NGX_HAVE_REUSEPORT) +#if(NGX_HAVE_REUSEPORT_CBPF) +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) +#endif + +static void attach_bpf(int fd, uint16_t n) +{ + struct sock_filter code[] = { + /* A = skb->queue_mapping */ + { BPF_LD | BPF_W | BPF_ABS, 0, 0, SKF_AD_OFF + SKF_AD_QUEUE }, + /* A = A % n */ + { BPF_ALU | BPF_MOD, 0, 0, n }, + /* return A */ + { BPF_RET | BPF_A, 0, 0, 0 }, + }; + struct sock_fprog p = { + .len = ARRAY_SIZE(code), + .filter = code, + }; + + if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF, &p, sizeof(p))) + printf("failed to set SO_ATTACH_REUSEPORT_CBPF"); + else + printf("bpf prog attached to fd:%d\n", fd); +} +#endif +#endif + void ngx_configure_listening_sockets(ngx_cycle_t *cycle) @@ -719,6 +751,11 @@ #if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER) struct accept_filter_arg af; #endif +#if (NGX_HAVE_REUSEPORT) +#if (NGX_HAVE_REUSEPORT_CBPF) + ngx_core_conf_t* ccf ; +#endif +#endif ls = cycle->listening.elts; for (i = 0; i < cycle->listening.nelts; i++) { @@ -1011,6 +1048,16 @@ } #endif +#if (NGX_HAVE_REUSEPORT) +#if (NGX_HAVE_REUSEPORT_CBPF) + if(ls[i].reuseport) + { + ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx,ngx_core_module); + if(ccf) + attach_bpf(ls[i].fd, ccf->worker_processes); + } +#endif +#endif } return;

2 years

HTTP/3 with multiple server blocks

by Jonny Barnes

Can we do this? First issue is if I add the line `listen 443 http3 resueport` to more than one server {} block the nginx conf test gives the following error: > nginx: [emerg] duplicate listen options for 0.0.0.0:443 in /usr/local/nginx/conf/sites-enabled/legolas:2 > nginx: configuration file /usr/local/nginx/conf/nginx.conf test failed It seemingly works if i only have it for one of my server blocks. But I can‘t get my firefox browser to connect with HTTP/3. It does connect to quic.nginx.org over HTTP/3.

2 years, 1 month

Nginx HTTP/3

by HARISH KUMAR Ivaturi

Hi Does anyone know how to include Nginx HTTP/3 support in Devstack? BR Harish Kumar

2 years, 1 month

[njs] Types: refactored ts_test.

by Dmitry Volyntsev

details: https://hg.nginx.org/njs/rev/fa3ffb3a159e branches: changeset: 1555:fa3ffb3a159e user: Jakub Jirutka <jakub(a)jirutka.cz> date: Thu Oct 29 12:51:21 2020 +0000 description: Types: refactored ts_test. diffstat: auto/make | 19 +++++++++++++++++-- auto/sources | 2 ++ test/ts/package.json | 15 +++++++++++++++ test/ts/test.ts | 22 +++++++++------------- test/ts/tsconfig.json | 35 +++++++++++++++++++++++++++++++++++ 5 files changed, 78 insertions(+), 15 deletions(-) diffs (172 lines): diff -r a3ae0d653f49 -r fa3ffb3a159e auto/make --- a/auto/make Thu Oct 29 12:50:05 2020 +0000 +++ b/auto/make Thu Oct 29 12:51:21 2020 +0000 @@ -248,6 +248,9 @@ END njs_ts_deps=`echo $NJS_TS_SRCS \ | sed -e "s# *$[^ ][^ ]*$#\1$njs_regex_cont#g"` +njs_test_ts_deps=`echo $NJS_TEST_TS_SRCS \ + | sed -e "s# *$[^ ][^ ]*$#\1$njs_regex_cont#g"` + cat << END >> $NJS_MAKEFILE $NJS_BUILD_DIR/ts/package.json: $njs_ts_deps @@ -270,8 +273,20 @@ ts: $NJS_BUILD_DIR/ts/package.json ts_lint: $NJS_BUILD_DIR/ts/node_modules cd $NJS_BUILD_DIR/ts && \$(NPM) run lint -ts_test: ts - tsc ./test/ts/test.ts +$NJS_BUILD_DIR/test/ts/package.json: $njs_test_ts_deps + mkdir -p $NJS_BUILD_DIR/test + cp -fr test/ts $NJS_BUILD_DIR/test/ + +$NJS_BUILD_DIR/test/ts/node_modules: \\ + $NJS_BUILD_DIR/njs-types-\$(NJS_TYPES_VER).tgz \\ + $NJS_BUILD_DIR/test/ts/package.json + cd $NJS_BUILD_DIR/test/ts && \$(NPM) install \\ + --save-dev file:../../njs-types-\$(NJS_TYPES_VER).tgz + cd $NJS_BUILD_DIR/test/ts && \$(NPM) install + touch $NJS_BUILD_DIR/test/ts/node_modules + +ts_test: $NJS_BUILD_DIR/test/ts/node_modules + cd $NJS_BUILD_DIR/test/ts && \$(NPM) test ts_publish: ts_clean $NJS_BUILD_DIR/njs-types-\$(NJS_TYPES_VER).tgz cd $NJS_BUILD_DIR/ && \$(NPM) publish njs-types-\$(NJS_TYPES_VER).tgz diff -r a3ae0d653f49 -r fa3ffb3a159e auto/sources --- a/auto/sources Thu Oct 29 12:50:05 2020 +0000 +++ b/auto/sources Thu Oct 29 12:51:21 2020 +0000 @@ -73,3 +73,5 @@ NJS_TEST_SRCS=" \ " NJS_TS_SRCS=$(find ts/ -name "*.d.ts" -o -name "*.json") + +NJS_TEST_TS_SRCS=$(find test/ts/ -name "*.ts" -o -name "*.json") diff -r a3ae0d653f49 -r fa3ffb3a159e test/ts/package.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/ts/package.json Thu Oct 29 12:51:21 2020 +0000 @@ -0,0 +1,15 @@ +{ + "private": true, + "name": "njs-types-test", + "version": "0.0.0", + "description": "Tests for njs TypeScript type definitions.", + "scripts": { + "test": "tsc" + }, + "author": "NGINX, Inc.", + "license": "BSD-2-Clause", + "devDependencies": { + "njs-types": "file:../../ts", + "typescript": "~4.0.3" + } +} diff -r a3ae0d653f49 -r fa3ffb3a159e test/ts/test.ts --- a/test/ts/test.ts Thu Oct 29 12:50:05 2020 +0000 +++ b/test/ts/test.ts Thu Oct 29 12:51:21 2020 +0000 @@ -1,8 +1,3 @@ -/// <reference path="../../build/ts/ngx_http_js_module.d.ts" /> -/// <reference path="../../build/ts/fs.d.ts" /> -/// <reference path="../../build/ts/querystring.d.ts" /> -/// <reference path="../../build/ts/crypto.d.ts" /> - import fs from 'fs'; import qs from 'querystring'; import crypto from 'crypto'; @@ -15,9 +10,9 @@ function http_module(r: NginxHTTPRequest s = 'ordinary string'; bs = String.bytesFrom('000000', 'hex'); - bs = s.toBytes(); + var bs2: NjsByteString | null = s.toBytes(); bs = s.toUTF8(); - bs.fromBytes(null, null); + bs.fromBytes(undefined, undefined); s = bs + ''; @@ -30,29 +25,30 @@ function http_module(r: NginxHTTPRequest bs = r.args.x; bs = r.args[1]; - s = r.args.x.fromUTF8(); + var s2: string | null = r.args.x.fromUTF8(); s = r.args.x + ''; // r.headersIn - r.headersIn['Accept'].fromBytes() == 'dddd'; + r.headersIn['Accept']?.fromBytes() == 'dddd'; // r.headersOut r.headersOut['Content-Type'] = 'text/plain'; // Warning: r.headersOut['Content-Type'] = ['a', 'b']; r.headersOut['Connection'] = undefined; - r.headersOut['Connection'] = null; + + delete r.headersOut['Bar']; r.headersOut['Set-Cookie'] = ['aaa', 'bbb']; r.headersOut['Foo'] = ['aaa', 'bbb']; - r.subrequest('/uri', reply => r.return(200, reply.headersOut["Location"])); + r.subrequest('/uri', reply => r.return(200, reply.headersOut["Location"] ?? '')); // r.log r.log(bs); - r.log(r.headersOut['Connection']); + r.log(r.headersOut['Connection'] ?? ''); // r.variables @@ -64,7 +60,7 @@ function http_module(r: NginxHTTPRequest r.subrequest('/p/sub2', reply => r.return(reply.status)); r.subrequest('/p/sub3', {detached:true}); r.subrequest('/p/sub4', 'a=1&b=2').then(reply => r.return(reply.status, - JSON.stringify(JSON.parse(reply.responseBody)))); + JSON.stringify(JSON.parse(reply.responseBody ?? '')))); } diff -r a3ae0d653f49 -r fa3ffb3a159e test/ts/tsconfig.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/ts/tsconfig.json Thu Oct 29 12:51:21 2020 +0000 @@ -0,0 +1,35 @@ +{ + "compilerOptions": { + "target": "ES5", + "module": "es2015", + "lib": [ + "ES2015", + "ES2016.Array.Include", + "ES2017.Object", + "ES2017.String" + ], + "noEmit": true, + "downlevelIteration": true, + + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "strictBindCallApply": true, + "strictPropertyInitialization": true, + "noImplicitThis": true, + "alwaysStrict": true, + + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true + }, + "include": [ + "**/*.ts" + ], + "exclude": [ + "./node_modules/**" + ], + "files": [ + "./node_modules/njs-types/ngx_http_js_module.d.ts" + ] +}

2 years, 1 month

[njs] Types: refactored working with TypeScript API description.

by Dmitry Volyntsev

details: https://hg.nginx.org/njs/rev/a3ae0d653f49 branches: changeset: 1554:a3ae0d653f49 user: Jakub Jirutka <jakub(a)jirutka.cz> date: Thu Oct 29 12:50:05 2020 +0000 description: Types: refactored working with TypeScript API description. diffstat: auto/make | 47 +++++++++++++++++++++----- auto/sources | 2 + ts/README.md | 79 ++++++++++++++++++++++++++++++++++++++++++++ ts/index.d.ts | 4 ++ ts/ngx_http_js_module.d.ts | 2 +- ts/ngx_stream_js_module.d.ts | 2 +- ts/njs_shell.d.ts | 2 +- ts/package.json | 3 +- 8 files changed, 128 insertions(+), 13 deletions(-) diffs (211 lines): diff -r 3cc202e562e4 -r a3ae0d653f49 auto/make --- a/auto/make Mon Oct 19 23:18:22 2020 +0200 +++ b/auto/make Thu Oct 29 12:50:05 2020 +0000 @@ -22,6 +22,9 @@ NJS_LINK = ${CC} ${NJS_LD_OPT} NJS_CFLAGS = ${NJS_CFLAGS} ${NJS_CC_OPT} ${CFLAGS} NJS_VER = $(grep NJS_VERSION src/njs.h | sed -e 's/.*"$.*$".*/\1/') +NJS_TYPES_VER = \$(NJS_VER) + +NPM = npm default: $NJS_DEFAULT_TARGET @@ -234,15 +237,6 @@ benchmark: $NJS_BUILD_DIR/njs_auto_confi $NJS_BUILD_DIR/njs_benchmark -.PHONY: ts -ts: - mkdir -p $NJS_BUILD_DIR/ts/njs_modules - cp ts/*.d.ts $NJS_BUILD_DIR/ts/ - cp ts/njs_modules/* $NJS_BUILD_DIR/ts/njs_modules/ - -ts_test: ts - tsc ./test/ts/test.ts - dist: rm -rf njs-\$(NJS_VER) \\ && hg archive njs-\$(NJS_VER).tar.gz \\ @@ -251,6 +245,41 @@ dist: && echo njs-\$(NJS_VER).tar.gz done END +njs_ts_deps=`echo $NJS_TS_SRCS \ + | sed -e "s# *$[^ ][^ ]*$#\1$njs_regex_cont#g"` + +cat << END >> $NJS_MAKEFILE + +$NJS_BUILD_DIR/ts/package.json: $njs_ts_deps + cp -fr ts $NJS_BUILD_DIR/ + cp LICENSE $NJS_BUILD_DIR/ts/ + sed 's/$"version":\s*$"$[^"]\+$"/\1"\$(NJS_TYPES_VER)"/' \\ + ts/package.json > $NJS_BUILD_DIR/ts/package.json + +$NJS_BUILD_DIR/ts/node_modules: $NJS_BUILD_DIR/ts/package.json + cd $NJS_BUILD_DIR/ts && \$(NPM) install + touch $NJS_BUILD_DIR/ts/node_modules + +$NJS_BUILD_DIR/njs-types-\$(NJS_TYPES_VER).tgz: $NJS_BUILD_DIR/ts/package.json + hg id -i > $NJS_BUILD_DIR/ts/COMMITHASH || true + cd $NJS_BUILD_DIR && \$(NPM) pack ./ts + +.PHONY: ts +ts: $NJS_BUILD_DIR/ts/package.json + +ts_lint: $NJS_BUILD_DIR/ts/node_modules + cd $NJS_BUILD_DIR/ts && \$(NPM) run lint + +ts_test: ts + tsc ./test/ts/test.ts + +ts_publish: ts_clean $NJS_BUILD_DIR/njs-types-\$(NJS_TYPES_VER).tgz + cd $NJS_BUILD_DIR/ && \$(NPM) publish njs-types-\$(NJS_TYPES_VER).tgz + +ts_clean: + rm -rf $NJS_BUILD_DIR/ts +END + # Makefile. diff -r 3cc202e562e4 -r a3ae0d653f49 auto/sources --- a/auto/sources Mon Oct 19 23:18:22 2020 +0200 +++ b/auto/sources Thu Oct 29 12:50:05 2020 +0000 @@ -71,3 +71,5 @@ NJS_TEST_SRCS=" \ src/test/njs_unit_test.c \ src/test/njs_benchmark.c \ " + +NJS_TS_SRCS=$(find ts/ -name "*.d.ts" -o -name "*.json") diff -r 3cc202e562e4 -r a3ae0d653f49 ts/README.md --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/ts/README.md Thu Oct 29 12:50:05 2020 +0000 @@ -0,0 +1,79 @@ +# TypeScript definitions for njs + +This package contains type definitions for [njs](https://github.com/nginx/njs) – NGINX JavaScript. + + +## Usage + +Install **njs-types** from the [npm registry](https://www.npmjs.com/) into your project: + +```sh +# using npm: +npm install --save-dev njs-types +# or using yarn: +yarn add --dev njs-types +``` + +njs-types provides three entry points with global declarations for each of njs environments: + +* `njs_shell.d.ts` – njs shell +* `ngx_http_js_module.d.ts` – NGINX JS HTTP Module +* `ngx_stream_js_module.d.ts` – NGINX JS Stream Module + +You can either reference them using [triple-slash directive](https://www.typescriptlang.org/docs/handbook/triple-slash-direct… at top of your `.ts` files (adjust `path` to point into your project’s node_modules): + +```ts +/// <reference path="./node_modules/njs-types/ngx_http_js_module.d.ts" /> +``` + +or include them using the [files](https://www.typescriptlang.org/tsconfig#files) flag in your `tsconfig.json`, for example: + +```json +{ + "compilerOptions": { + "target": "ES5", + "module": "es2015", + "lib": [ + "ES2015", + "ES2016.Array.Include", + "ES2017.Object", + "ES2017.String" + ], + "outDir": "./lib", + "downlevelIteration": true, + + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "strictBindCallApply": true, + "strictPropertyInitialization": true, + "noImplicitThis": true, + "alwaysStrict": true, + + "moduleResolution": "node", + + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + }, + "include": [ + "./src", + ], + "files": [ + "./node_modules/njs-types/ngx_http_js_module.d.ts", + ], +} +``` + + +## Versions + +njs-types is typically being released together with njs. +Their major and minor release numbers (the first two numbers) are always aligned, but the patch version (the third number) may differ. +That's because njs-types may be updated between njs releases and in such case the patch version is incremented. + +It's the same strategy as used in [DefinitelyTyped](https://github.com/DefinitelyTyped/DefinitelyTyped#how-do-…. +The reason is that npmjs enforces [SemVer](https://semver.org/) which doesn't allow four-part version number nor provide post-release suffixes. + +You can find from which commit the package was built in file `COMMITHASH` inside the published package. +It contains global revision id in the upstream repository https://hg.nginx.org/njs/ (Mercurial). diff -r 3cc202e562e4 -r a3ae0d653f49 ts/index.d.ts --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/ts/index.d.ts Thu Oct 29 12:50:05 2020 +0000 @@ -0,0 +1,4 @@ +/// <reference path="njs_core.d.ts" /> +/// <reference path="njs_modules/crypto.d.ts" /> +/// <reference path="njs_modules/fs.d.ts" /> +/// <reference path="njs_modules/querystring.d.ts" /> diff -r 3cc202e562e4 -r a3ae0d653f49 ts/ngx_http_js_module.d.ts --- a/ts/ngx_http_js_module.d.ts Mon Oct 19 23:18:22 2020 +0200 +++ b/ts/ngx_http_js_module.d.ts Thu Oct 29 12:50:05 2020 +0000 @@ -1,4 +1,4 @@ -/// <reference path="njs_core.d.ts" /> +/// <reference path="index.d.ts" /> interface NginxHTTPArgs { readonly [prop: string]: NjsByteString; diff -r 3cc202e562e4 -r a3ae0d653f49 ts/ngx_stream_js_module.d.ts --- a/ts/ngx_stream_js_module.d.ts Mon Oct 19 23:18:22 2020 +0200 +++ b/ts/ngx_stream_js_module.d.ts Thu Oct 29 12:50:05 2020 +0000 @@ -1,4 +1,4 @@ -/// <reference path="njs_core.d.ts" /> +/// <reference path="index.d.ts" /> interface NginxStreamVariables { readonly 'binary_remote_addr'?: NjsByteString; diff -r 3cc202e562e4 -r a3ae0d653f49 ts/njs_shell.d.ts --- a/ts/njs_shell.d.ts Mon Oct 19 23:18:22 2020 +0200 +++ b/ts/njs_shell.d.ts Thu Oct 29 12:50:05 2020 +0000 @@ -1,4 +1,4 @@ -/// <reference path="njs_core.d.ts" /> +/// <reference path="index.d.ts" /> interface Console { log(...args: any[]): void; diff -r 3cc202e562e4 -r a3ae0d653f49 ts/package.json --- a/ts/package.json Mon Oct 19 23:18:22 2020 +0200 +++ b/ts/package.json Thu Oct 29 12:50:05 2020 +0000 @@ -6,7 +6,8 @@ "lint": "tsc" }, "files": [ - "**/*.d.ts" + "**/*.d.ts", + "COMMITHASH" ], "types": "index.d.ts", "repository": {

2 years, 1 month

[njs] Types: fixed index types in modules.

by Dmitry Volyntsev

details: https://hg.nginx.org/njs/rev/3cc202e562e4 branches: changeset: 1553:3cc202e562e4 user: Jakub Jirutka <jakub(a)jirutka.cz> date: Mon Oct 19 23:18:22 2020 +0200 description: Types: fixed index types in modules. Fixes the following type errors: Property ''xxx'' of type 'NjsByteString | undefined' is not assignable to string index type 'NjsStringLike'.ts(2411) While string index signatures are a powerful way to describe the "dictionary" pattern, they also enforce that all properties match their return type. diffstat: ts/ngx_http_js_module.d.ts | 6 +++--- ts/ngx_stream_js_module.d.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diffs (42 lines): diff -r 862ff6f2c684 -r 3cc202e562e4 ts/ngx_http_js_module.d.ts --- a/ts/ngx_http_js_module.d.ts Sat Oct 24 23:59:18 2020 +0200 +++ b/ts/ngx_http_js_module.d.ts Mon Oct 19 23:18:22 2020 +0200 @@ -39,7 +39,7 @@ interface NginxHeadersIn { readonly 'Warning'?: NjsByteString; readonly 'X-Forwarded-For'?: NjsByteString; - readonly [prop: string]: NjsByteString; + readonly [prop: string]: NjsByteString | undefined; } interface NginxHeadersOut { @@ -76,7 +76,7 @@ interface NginxHeadersOut { 'Set-Cookie'?: NjsStringLike[]; - [prop: string]: NjsStringLike | NjsStringLike[]; + [prop: string]: NjsStringLike | NjsStringLike[] | undefined; } interface NginxVariables { @@ -229,7 +229,7 @@ interface NginxVariables { readonly 'upstream_trailer_'?: NjsByteString; readonly 'uri'?: NjsByteString; - [prop: string]: NjsStringLike; + [prop: string]: NjsStringLike | undefined; } interface NginxSubrequestOptions { diff -r 862ff6f2c684 -r 3cc202e562e4 ts/ngx_stream_js_module.d.ts --- a/ts/ngx_stream_js_module.d.ts Sat Oct 24 23:59:18 2020 +0200 +++ b/ts/ngx_stream_js_module.d.ts Mon Oct 19 23:18:22 2020 +0200 @@ -66,7 +66,7 @@ interface NginxStreamVariables { readonly 'time_iso8601'?: NjsByteString; readonly 'time_local'?: NjsByteString; - [prop: string]: NjsByteString; + [prop: string]: NjsByteString | undefined; } interface NginxStreamCallbackFlags {

2 years, 1 month

[njs] Types: added missing var "console" into njs_shell.

by Dmitry Volyntsev

details: https://hg.nginx.org/njs/rev/862ff6f2c684 branches: changeset: 1552:862ff6f2c684 user: Jakub Jirutka <jakub(a)jirutka.cz> date: Sat Oct 24 23:59:18 2020 +0200 description: Types: added missing var "console" into njs_shell. diffstat: ts/njs_shell.d.ts | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (9 lines): diff -r e1dc1fd52fe9 -r 862ff6f2c684 ts/njs_shell.d.ts --- a/ts/njs_shell.d.ts Tue Oct 20 21:16:20 2020 +0200 +++ b/ts/njs_shell.d.ts Sat Oct 24 23:59:18 2020 +0200 @@ -7,3 +7,5 @@ interface Console { time(label?: NjsStringLike): void; timeEnd(label?: NjsStringLike): void; } + +declare const console: Console;

2 years, 1 month

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

nginx-devel October 2020