nginx-devel May 2020

nginx-devel@nginx.org

20 participants
58 discussions

[PATCH] Added support for proxying managesieve protocol

by Sander Hoentjen

# HG changeset patch # User Sander Hoentjen <shoentjen(a)antagonist.nl> # Date 1586369831 -7200 # Wed Apr 08 20:17:11 2020 +0200 # Node ID f1dffaf619688aaab90caf31781ebe27c3f79598 # Parent 0cb942c1c1aa98118076e72e0b89940e85e6291c Added support for proxying managesieve protocol Sieve is a widely used protocol to implement server-side filtering. Some servers and clients that support this are listed at <http://sieve.info/>. This commit is tested with Dovecot as the server and Roundcube as the client, but I don't see a readon why it not would work with others. It is a nice addition to nginx because it adds the possibility to add server side filtering to setups that use IMAP behind an nginx proxy. Fixes https://trac.nginx.org/nginx/ticket/1697 diff -r 0cb942c1c1aa -r f1dffaf61968 auto/modules --- a/auto/modules Fri Mar 13 02:12:10 2020 +0300 +++ b/auto/modules Wed Apr 08 20:17:11 2020 +0200 @@ -965,6 +965,15 @@ . auto/module fi + if [ $MAIL_SIEVE = YES ]; then + ngx_module_name=ngx_mail_sieve_module + ngx_module_deps=src/mail/ngx_mail_sieve_module.h + ngx_module_srcs="src/mail/ngx_mail_sieve_module.c \ + src/mail/ngx_mail_sieve_handler.c" + + . auto/module + fi + if [ $MAIL_SMTP = YES ]; then ngx_module_name=ngx_mail_smtp_module ngx_module_deps=src/mail/ngx_mail_smtp_module.h diff -r 0cb942c1c1aa -r f1dffaf61968 auto/options --- a/auto/options Fri Mar 13 02:12:10 2020 +0300 +++ b/auto/options Wed Apr 08 20:17:11 2020 +0200 @@ -112,6 +112,7 @@ MAIL_SSL=NO MAIL_POP3=YES MAIL_IMAP=YES +MAIL_SIEVE=YES MAIL_SMTP=YES STREAM=NO @@ -305,6 +306,8 @@ ;; --without-mail_pop3_module) MAIL_POP3=NO ;; --without-mail_imap_module) MAIL_IMAP=NO ;; + --without-mail_sieve_module) + MAIL_SIEVE=NO ;; --without-mail_smtp_module) MAIL_SMTP=NO ;; --with-stream) STREAM=YES ;; @@ -517,11 +520,12 @@ --without-http disable HTTP server --without-http-cache disable HTTP cache - --with-mail enable POP3/IMAP4/SMTP proxy module - --with-mail=dynamic enable dynamic POP3/IMAP4/SMTP proxy module + --with-mail enable POP3/IMAP4/SIEVE/SMTP proxy module + --with-mail=dynamic enable dynamic POP3/IMAP4/SIEVE/SMTP proxy module --with-mail_ssl_module enable ngx_mail_ssl_module --without-mail_pop3_module disable ngx_mail_pop3_module --without-mail_imap_module disable ngx_mail_imap_module + --without-mail_sieve_module disable ngx_mail_sieve_module --without-mail_smtp_module disable ngx_mail_smtp_module --with-stream enable TCP/UDP proxy module diff -r 0cb942c1c1aa -r f1dffaf61968 contrib/vim/syntax/nginx.vim --- a/contrib/vim/syntax/nginx.vim Fri Mar 13 02:12:10 2020 +0300 +++ b/contrib/vim/syntax/nginx.vim Wed Apr 08 20:17:11 2020 +0200 @@ -571,6 +571,9 @@ syn keyword ngxDirective contained session_log_format syn keyword ngxDirective contained session_log_zone syn keyword ngxDirective contained set_real_ip_from +syn keyword ngxDirective contained sieve_auth +syn keyword ngxDirective contained sieve_capabilities +syn keyword ngxDirective contained sieve_client_buffer syn keyword ngxDirective contained slice syn keyword ngxDirective contained smtp_auth syn keyword ngxDirective contained smtp_capabilities diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail.h --- a/src/mail/ngx_mail.h Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail.h Wed Apr 08 20:17:11 2020 +0200 @@ -102,6 +102,7 @@ #define NGX_MAIL_POP3_PROTOCOL 0 #define NGX_MAIL_IMAP_PROTOCOL 1 #define NGX_MAIL_SMTP_PROTOCOL 2 +#define NGX_MAIL_SIEVE_PROTOCOL 3 typedef struct ngx_mail_protocol_s ngx_mail_protocol_t; @@ -154,6 +155,21 @@ typedef enum { + ngx_sieve_start = 0, + ngx_sieve_starttls, + ngx_sieve_auth_login_username, + ngx_sieve_auth_login_password, + ngx_sieve_auth_plain, + ngx_sieve_auth_cram_md5, + ngx_sieve_auth_external, + ngx_sieve_login, + ngx_sieve_login_capabilities, + ngx_sieve_user, + ngx_sieve_passwd +} ngx_sieve_state_e; + + +typedef enum { ngx_smtp_start = 0, ngx_smtp_auth_login_username, ngx_smtp_auth_login_password, @@ -227,7 +243,7 @@ ngx_uint_t login_attempt; - /* used to parse POP3/IMAP/SMTP command */ + /* used to parse POP3/IMAP/SIEVE/SMTP command */ ngx_uint_t state; u_char *cmd_start; @@ -271,6 +287,14 @@ #define NGX_IMAP_AUTHENTICATE 7 +#define NGX_SIEVE_LOGOUT 1 +#define NGX_SIEVE_CAPABILITY 2 +#define NGX_SIEVE_NOOP 3 +#define NGX_SIEVE_STARTTLS 4 +#define NGX_SIEVE_NEXT 5 +#define NGX_SIEVE_AUTHENTICATE 6 + + #define NGX_SMTP_HELO 1 #define NGX_SMTP_EHLO 2 #define NGX_SMTP_AUTH 3 @@ -383,7 +407,7 @@ ngx_int_t ngx_mail_auth_login_password(ngx_mail_session_t *s, ngx_connection_t *c); ngx_int_t ngx_mail_auth_cram_md5_salt(ngx_mail_session_t *s, - ngx_connection_t *c, char *prefix, size_t len); + ngx_connection_t *c, char *prefix, size_t len, ngx_uint_t quote); ngx_int_t ngx_mail_auth_cram_md5(ngx_mail_session_t *s, ngx_connection_t *c); ngx_int_t ngx_mail_auth_external(ngx_mail_session_t *s, ngx_connection_t *c, ngx_uint_t n); diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_auth_http_module.c --- a/src/mail/ngx_mail_auth_http_module.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_auth_http_module.c Wed Apr 08 20:17:11 2020 +0200 @@ -533,6 +533,11 @@ + sizeof(CRLF) - 1; break; + case NGX_MAIL_SIEVE_PROTOCOL: + size = sizeof("BYE \"\"") - 1 + len + + sizeof(CRLF) - 1; + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ ctx->err = ctx->errmsg; continue; @@ -559,11 +564,27 @@ *p++ = 'N'; *p++ = 'O'; *p++ = ' '; break; + case NGX_MAIL_SIEVE_PROTOCOL: + p = ngx_cpymem(p, s->tag.data, s->tag.len); + *p++ = 'B'; *p++ = 'Y'; *p++ = 'E'; *p++ = ' '; *p++ = '"'; + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ break; } p = ngx_cpymem(p, ctx->header_start, len); + + switch (s->protocol) { + + case NGX_MAIL_SIEVE_PROTOCOL: + *p++ = '"'; + break; + + default: + break; + } + *p++ = CR; *p++ = LF; ctx->err.len = p - ctx->err.data; diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_handler.c --- a/src/mail/ngx_mail_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -520,28 +520,39 @@ ngx_int_t ngx_mail_auth_cram_md5_salt(ngx_mail_session_t *s, ngx_connection_t *c, - char *prefix, size_t len) + char *prefix, size_t len, ngx_uint_t quote) { u_char *p; ngx_str_t salt; - ngx_uint_t n; + ngx_uint_t n = 0; + if (quote) { + len += 2; + } p = ngx_pnalloc(c->pool, len + ngx_base64_encoded_length(s->salt.len) + 2); if (p == NULL) { return NGX_ERROR; } - salt.data = ngx_cpymem(p, prefix, len); + if (quote) { + len -= 2; + salt.data = ngx_cpymem(p, "\"", 1); + n++; + } + salt.data = ngx_cpymem(p+ n, prefix, len); s->salt.len -= 2; ngx_encode_base64(&salt, &s->salt); s->salt.len += 2; - n = len + salt.len; + n += len + salt.len; + if (quote) { + p[n++] = '"'; + } p[n++] = CR; p[n++] = LF; s->out.len = n; - s->out.data = p; + s->out.data = p--; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_imap_handler.c --- a/src/mail/ngx_mail_imap_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_imap_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -397,7 +397,7 @@ } } - if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2) == NGX_OK) { + if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2, 0) == NGX_OK) { s->mail_state = ngx_imap_auth_cram_md5; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_parse.c --- a/src/mail/ngx_mail_parse.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_parse.c Wed Apr 08 20:17:11 2020 +0200 @@ -11,6 +11,7 @@ #include <ngx_mail.h> #include <ngx_mail_pop3_module.h> #include <ngx_mail_imap_module.h> +#include <ngx_mail_sieve_module.h> #include <ngx_mail_smtp_module.h> @@ -620,6 +621,364 @@ ngx_int_t +ngx_mail_sieve_parse_command(ngx_mail_session_t *s) +{ + u_char ch, *p, *c; + ngx_str_t *arg; + enum { + sw_start = 0, + sw_spaces_before_argument, + sw_argument, + sw_backslash, + sw_literal, + sw_no_sync_literal_argument, + sw_start_literal_argument, + sw_literal_argument, + sw_end_literal_argument, + sw_almost_done + } state; + + state = s->state; + + for (p = s->buffer->pos; p < s->buffer->last; p++) { + ch = *p; + + switch (state) { + + /* SIEVE command */ + case sw_start: + if (ch == ' ' || ch == CR || ch == LF) { + + c = s->buffer->start; + + switch (p - c) { + + case 4: + if ((c[0] == 'N' || c[0] == 'n') + && (c[1] == 'O'|| c[1] == 'o') + && (c[2] == 'O'|| c[2] == 'o') + && (c[3] == 'P'|| c[3] == 'p')) + { + s->command = NGX_SIEVE_NOOP; + + } else { + goto invalid; + } + break; + + case 6: + if ((c[0] == 'L'|| c[0] == 'l') + && (c[1] == 'O'|| c[1] == 'o') + && (c[2] == 'G'|| c[2] == 'g') + && (c[3] == 'O'|| c[3] == 'o') + && (c[4] == 'U'|| c[4] == 'u') + && (c[5] == 'T'|| c[5] == 't')) + { + s->command = NGX_SIEVE_LOGOUT; + + } else { + goto invalid; + } + break; + +#if (NGX_MAIL_SSL) + case 8: + if ((c[0] == 'S'|| c[0] == 's') + && (c[1] == 'T'|| c[1] == 't') + && (c[2] == 'A'|| c[2] == 'a') + && (c[3] == 'R'|| c[3] == 'r') + && (c[4] == 'T'|| c[4] == 't') + && (c[5] == 'T'|| c[5] == 't') + && (c[6] == 'L'|| c[6] == 'l') + && (c[7] == 'S'|| c[7] == 's')) + { + s->command = NGX_SIEVE_STARTTLS; + + } else { + goto invalid; + } + break; +#endif + + case 10: + if ((c[0] == 'C'|| c[0] == 'c') + && (c[1] == 'A'|| c[1] == 'a') + && (c[2] == 'P'|| c[2] == 'p') + && (c[3] == 'A'|| c[3] == 'a') + && (c[4] == 'B'|| c[4] == 'b') + && (c[5] == 'I'|| c[5] == 'i') + && (c[6] == 'L'|| c[6] == 'l') + && (c[7] == 'I'|| c[7] == 'i') + && (c[8] == 'T'|| c[8] == 't') + && (c[9] == 'Y'|| c[9] == 'y')) + { + s->command = NGX_SIEVE_CAPABILITY; + + } else { + goto invalid; + } + break; + + case 12: + if ((c[0] == 'A'|| c[0] == 'a') + && (c[1] == 'U'|| c[1] == 'u') + && (c[2] == 'T'|| c[2] == 't') + && (c[3] == 'H'|| c[3] == 'h') + && (c[4] == 'E'|| c[4] == 'e') + && (c[5] == 'N'|| c[5] == 'n') + && (c[6] == 'T'|| c[6] == 't') + && (c[7] == 'I'|| c[7] == 'i') + && (c[8] == 'C'|| c[8] == 'c') + && (c[9] == 'A'|| c[9] == 'a') + && (c[10] == 'T'|| c[10] == 't') + && (c[11] == 'E'|| c[11] == 'e')) + { + s->command = NGX_SIEVE_AUTHENTICATE; + + } else { + goto invalid; + } + break; + + default: + goto invalid; + } + + switch (ch) { + case ' ': + state = sw_spaces_before_argument; + break; + case CR: + state = sw_almost_done; + break; + case LF: + goto done; + } + break; + } + + if ((ch < 'A' || ch > 'Z') && (ch < 'a' || ch > 'z')) { + goto invalid; + } + + break; + + case sw_spaces_before_argument: + switch (ch) { + case ' ': + break; + case CR: + state = sw_almost_done; + s->arg_end = p; + break; + case LF: + s->arg_end = p; + goto done; + case '"': + if (s->args.nelts <= 2) { + s->quoted = 1; + s->arg_start = p + 1; + state = sw_argument; + break; + } + goto invalid; + case '{': + if (s->args.nelts <= 2) { + state = sw_literal; + break; + } + goto invalid; + default: + if (s->args.nelts <= 2) { + s->arg_start = p; + state = sw_argument; + break; + } + goto invalid; + } + break; + + case sw_argument: + if (ch == ' ' && s->quoted) { + break; + } + + switch (ch) { + case '"': + if (!s->quoted) { + break; + } + s->quoted = 0; + /* fall through */ + case ' ': + case CR: + case LF: + arg = ngx_array_push(&s->args); + if (arg == NULL) { + return NGX_ERROR; + } + arg->len = p - s->arg_start; + arg->data = s->arg_start; + s->arg_start = NULL; + + switch (ch) { + case '"': + case ' ': + state = sw_spaces_before_argument; + break; + case CR: + state = sw_almost_done; + break; + case LF: + goto done; + } + break; + case '\\': + if (s->quoted) { + s->backslash = 1; + state = sw_backslash; + } + break; + } + break; + + case sw_backslash: + switch (ch) { + case CR: + case LF: + goto invalid; + default: + state = sw_argument; + } + break; + + case sw_literal: + if (ch >= '0' && ch <= '9') { + s->literal_len = s->literal_len * 10 + (ch - '0'); + break; + } + if (ch == '}') { + state = sw_start_literal_argument; + break; + } + if (ch == '+') { + state = sw_no_sync_literal_argument; + break; + } + goto invalid; + + case sw_no_sync_literal_argument: + if (ch == '}') { + s->no_sync_literal = 1; + state = sw_start_literal_argument; + break; + } + goto invalid; + + case sw_start_literal_argument: + switch (ch) { + case CR: + break; + case LF: + s->buffer->pos = p + 1; + s->arg_start = p + 1; + if (s->no_sync_literal == 0) { + s->state = sw_literal_argument; + return NGX_SIEVE_NEXT; + } + state = sw_literal_argument; + s->no_sync_literal = 0; + break; + default: + goto invalid; + } + break; + + case sw_literal_argument: + if (s->literal_len && --s->literal_len) { + break; + } + + arg = ngx_array_push(&s->args); + if (arg == NULL) { + return NGX_ERROR; + } + arg->len = p + 1 - s->arg_start; + arg->data = s->arg_start; + s->arg_start = NULL; + state = sw_end_literal_argument; + + break; + + case sw_end_literal_argument: + switch (ch) { + case '{': + if (s->args.nelts <= 2) { + state = sw_literal; + break; + } + goto invalid; + case CR: + state = sw_almost_done; + break; + case LF: + goto done; + default: + state = sw_spaces_before_argument; + break; + } + break; + + case sw_almost_done: + switch (ch) { + case LF: + goto done; + default: + goto invalid; + } + } + } + + s->buffer->pos = p; + s->state = state; + + return NGX_AGAIN; + +done: + + s->buffer->pos = p + 1; + + if (s->arg_start) { + arg = ngx_array_push(&s->args); + if (arg == NULL) { + return NGX_ERROR; + } + arg->len = s->arg_end - s->arg_start; + arg->data = s->arg_start; + + s->arg_start = NULL; + s->cmd_start = NULL; + s->quoted = 0; + s->no_sync_literal = 0; + s->literal_len = 0; + } + + s->state = (s->command != NGX_SIEVE_AUTHENTICATE) ? sw_start : sw_spaces_before_argument; + + return NGX_OK; + +invalid: + + s->state = sw_start; + s->quoted = 0; + s->no_sync_literal = 0; + s->literal_len = 0; + + return NGX_MAIL_PARSE_INVALID_COMMAND; +} + + +ngx_int_t ngx_mail_smtp_parse_command(ngx_mail_session_t *s) { u_char ch, *p, *c, c0, c1, c2, c3; diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_pop3_handler.c --- a/src/mail/ngx_mail_pop3_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_pop3_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -492,7 +492,7 @@ return NGX_MAIL_PARSE_INVALID_COMMAND; } - if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2) == NGX_OK) { + if (ngx_mail_auth_cram_md5_salt(s, c, "+ ", 2, 0) == NGX_OK) { s->mail_state = ngx_pop3_auth_cram_md5; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_proxy_module.c --- a/src/mail/ngx_mail_proxy_module.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_proxy_module.c Wed Apr 08 20:17:11 2020 +0200 @@ -24,6 +24,7 @@ static void ngx_mail_proxy_block_read(ngx_event_t *rev); static void ngx_mail_proxy_pop3_handler(ngx_event_t *rev); static void ngx_mail_proxy_imap_handler(ngx_event_t *rev); +static void ngx_mail_proxy_sieve_handler(ngx_event_t *rev); static void ngx_mail_proxy_smtp_handler(ngx_event_t *rev); static void ngx_mail_proxy_dummy_handler(ngx_event_t *ev); static ngx_int_t ngx_mail_proxy_read_response(ngx_mail_session_t *s, @@ -173,6 +174,11 @@ s->mail_state = ngx_imap_start; break; + case NGX_MAIL_SIEVE_PROTOCOL: + p->upstream.connection->read->handler = ngx_mail_proxy_sieve_handler; + s->mail_state = ngx_sieve_start; + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ p->upstream.connection->read->handler = ngx_mail_proxy_smtp_handler; s->mail_state = ngx_smtp_start; @@ -446,6 +452,144 @@ static void +ngx_mail_proxy_sieve_handler(ngx_event_t *rev) +{ + u_char *p; + ngx_int_t rc; + ngx_str_t line; + ngx_str_t userpass; + ngx_str_t b64_userpass; + ngx_connection_t *c; + ngx_mail_session_t *s; + ngx_mail_proxy_conf_t *pcf; + + ngx_log_debug0(NGX_LOG_DEBUG_MAIL, rev->log, 0, + "mail proxy sieve auth handler"); + + c = rev->data; + s = c->data; + + if (rev->timedout) { + ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, + "upstream timed out"); + c->timedout = 1; + ngx_mail_proxy_internal_server_error(s); + return; + } + + rc = ngx_mail_proxy_read_response(s, s->mail_state); + + if (rc == NGX_AGAIN) { + return; + } + + if (rc == NGX_ERROR) { + ngx_mail_proxy_upstream_error(s); + return; + } + + switch (s->mail_state) { + + case ngx_sieve_start: + ngx_log_debug0(NGX_LOG_DEBUG_MAIL, rev->log, 0, + "mail proxy send login"); + + s->connection->log->action = "sending LOGIN command to upstream"; + + line.len = s->tag.len + sizeof("AUTHENTICATE ") - 1 + + 1 + NGX_SIZE_T_LEN + 1 + 2; + line.data = ngx_pnalloc(c->pool, line.len); + if (line.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + + userpass.len = s->login.len + s->passwd.len + 2; + userpass.data = ngx_pnalloc(c->pool, userpass.len); + if (userpass.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + b64_userpass.len = ngx_base64_encoded_length(userpass.len) + 2; + b64_userpass.data = ngx_pnalloc(c->pool, b64_userpass.len); + if (b64_userpass.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + + p = userpass.data; + *p++ = '\0'; + p = ngx_cpymem(p, s->login.data, s->login.len); + *p++ = '\0'; + p = ngx_cpymem(p, s->passwd.data, s->passwd.len); + + ngx_encode_base64(&b64_userpass, &userpass); + line.len = ngx_sprintf(line.data, "AUTHENTICATE \"PLAIN\" \"%V\"" CRLF, + &b64_userpass) + - line.data; + + s->mail_state = ngx_sieve_login; + break; + + case ngx_sieve_login: + ngx_log_debug0(NGX_LOG_DEBUG_MAIL, rev->log, 0, + "mail proxy send capabilities"); + + s->connection->log->action = "sending CAPABILITY command to upstream"; + + line.len = s->tag.len + sizeof("AUTHENTICATE ") - 1 + + 1 + NGX_SIZE_T_LEN + 1 + 2; + line.data = ngx_pnalloc(c->pool, line.len); + if (line.data == NULL) { + ngx_mail_proxy_internal_server_error(s); + return; + } + + line.len = ngx_sprintf(line.data, "CAPABILITY" CRLF) + - line.data; + + s->mail_state = ngx_sieve_login_capabilities; + break; + + case ngx_sieve_login_capabilities: + s->connection->read->handler = ngx_mail_proxy_handler; + s->connection->write->handler = ngx_mail_proxy_handler; + rev->handler = ngx_mail_proxy_handler; + c->write->handler = ngx_mail_proxy_handler; + + pcf = ngx_mail_get_module_srv_conf(s, ngx_mail_proxy_module); + ngx_add_timer(s->connection->read, pcf->timeout); + ngx_del_timer(c->read); + + c->log->action = NULL; + ngx_log_error(NGX_LOG_INFO, c->log, 0, "client logged in"); + + ngx_mail_proxy_handler(s->connection->write); + + return; + + default: +#if (NGX_SUPPRESS_WARN) + ngx_str_null(&line); +#endif + break; + } + + if (c->send(c, line.data, line.len) < (ssize_t) line.len) { + /* + * we treat the incomplete sending as NGX_ERROR + * because it is very strange here + */ + ngx_mail_proxy_internal_server_error(s); + return; + } + + s->proxy->buffer->pos = s->proxy->buffer->start; + s->proxy->buffer->last = s->proxy->buffer->start; +} + + +static void ngx_mail_proxy_smtp_handler(ngx_event_t *rev) { u_char *p; @@ -793,6 +937,42 @@ break; + case NGX_MAIL_SIEVE_PROTOCOL: + if (p[0] == '"') { + m = b->last - (sizeof(CRLF "OK" CRLF) - 1); + + while (m > p) { + if (m[0] == CR && m[1] == LF) { + break; + } + m--; + } + + if (m <= p || m[0] == '"') { + return NGX_AGAIN; + } + p = m + 2; + } + + switch (state) { + + case ngx_sieve_start: + case ngx_sieve_login: + ngx_log_error(NGX_LOG_INFO, s->connection->log, 0, "1 upstream sent: \"%s\"", p); + if (p[0] == 'O' && p[1] == 'K') { + return NGX_OK; + } + break; + case ngx_sieve_login_capabilities: + ngx_log_error(NGX_LOG_INFO, s->connection->log, 0, "2 upstream sent: \"%s\"", p); + if (p[0] == 'O' && p[1] == 'K') { + return NGX_OK; + } + break; + } + + break; + default: /* NGX_MAIL_SMTP_PROTOCOL */ if (p[3] == '-') { diff -r 0cb942c1c1aa -r f1dffaf61968 src/mail/ngx_mail_smtp_handler.c --- a/src/mail/ngx_mail_smtp_handler.c Fri Mar 13 02:12:10 2020 +0300 +++ b/src/mail/ngx_mail_smtp_handler.c Wed Apr 08 20:17:11 2020 +0200 @@ -693,7 +693,7 @@ } } - if (ngx_mail_auth_cram_md5_salt(s, c, "334 ", 4) == NGX_OK) { + if (ngx_mail_auth_cram_md5_salt(s, c, "334 ", 4, 0) == NGX_OK) { s->mail_state = ngx_smtp_auth_cram_md5; return NGX_OK; } diff -r 0cb942c1c1aa -r f1dffaf61968 src/misc/ngx_cpp_test_module.cpp --- a/src/misc/ngx_cpp_test_module.cpp Fri Mar 13 02:12:10 2020 +0300 +++ b/src/misc/ngx_cpp_test_module.cpp Wed Apr 08 20:17:11 2020 +0200 @@ -13,6 +13,7 @@ #include <ngx_mail.h> #include <ngx_mail_pop3_module.h> #include <ngx_mail_imap_module.h> + #include <ngx_mail_sieve_module.h> #include <ngx_mail_smtp_module.h> }

10 months

Re: PCRE2 support?

by Maxim Dounin

Hello! On Tue, Sep 18, 2018 at 08:12:20AM -0400, Thomas Ward wrote: > Downstream in Ubuntu, it has been proposed to demote pcre3 and > use pcre2 instead as it is newer. > https://trac.nginx.org/nginx/ticket/720 shows it was marked 4 > years ago that NGINX does not support pcre2. Are there any > plans to use pcre2 instead of pcre3? There are no immediate plans. When we last checked, there were no problems with PCRE, but PCRE2 wasn't available in most distributions we support, making the switch mostly meaningless. Also, it looks like PCRE2 is still not supported even by Exim, which is the parent project of PCRE and PCRE2: https://bugs.exim.org/show_bug.cgi?id=1878 As such, adding PCRE2 support to nginx looks premature. -- Maxim Dounin http://mdounin.ru/

10 months

cache: move open to thread pool

by Ka-Hing Cheung

commit 0bcfefa040abdff674047c15701d237ff16a5f2b Author: Ka-Hing Cheung <kahing(a)cloudflare.com> Date: Fri Aug 3 13:37:58 2018 -0700 move open to thread pool At cloudflare we found that open() can block a long time, especially at p99 and p999. Moving it to thread pool improved overall p99 by 6x or so during peak time. This has the side effect of disabling nginx's open_file_cache when "aio threads" is turned on. For us we found that to have no impact (probably because we have way too many files for open file cache to be effective anyway). thread pool does increase CPU usage somewhat but we have not measured difference in CPU usage for stock "aio threads" compared to after this patch. Only the cache hit open() is moved to thread pool. We wrote more about it here: https://blog.cloudflare.com/how-we-scaled-nginx-and-saved-the-world-54-year… diff --git src/core/ngx_open_file_cache.c src/core/ngx_open_file_cache.c index b23ee78d..9177ebfd 100644 --- src/core/ngx_open_file_cache.c +++ src/core/ngx_open_file_cache.c @@ -35,8 +35,6 @@ static ngx_fd_t ngx_open_file_wrapper(ngx_str_t *name, ngx_int_t access, ngx_log_t *log); static ngx_int_t ngx_file_info_wrapper(ngx_str_t *name, ngx_open_file_info_t *of, ngx_file_info_t *fi, ngx_log_t *log); -static ngx_int_t ngx_open_and_stat_file(ngx_str_t *name, - ngx_open_file_info_t *of, ngx_log_t *log); static void ngx_open_file_add_event(ngx_open_file_cache_t *cache, ngx_cached_open_file_t *file, ngx_open_file_info_t *of, ngx_log_t *log); static void ngx_open_file_cleanup(void *data); @@ -836,7 +834,7 @@ ngx_file_info_wrapper(ngx_str_t *name, ngx_open_file_info_t *of, } -static ngx_int_t +ngx_int_t ngx_open_and_stat_file(ngx_str_t *name, ngx_open_file_info_t *of, ngx_log_t *log) { diff --git src/core/ngx_open_file_cache.h src/core/ngx_open_file_cache.h index d119c129..94b4d552 100644 --- src/core/ngx_open_file_cache.h +++ src/core/ngx_open_file_cache.h @@ -124,6 +124,8 @@ ngx_open_file_cache_t *ngx_open_file_cache_init(ngx_pool_t *pool, ngx_uint_t max, time_t inactive); ngx_int_t ngx_open_cached_file(ngx_open_file_cache_t *cache, ngx_str_t *name, ngx_open_file_info_t *of, ngx_pool_t *pool); +ngx_int_t ngx_open_and_stat_file(ngx_str_t *name, + ngx_open_file_info_t *of, ngx_log_t *log); #endif /* _NGX_OPEN_FILE_CACHE_H_INCLUDED_ */ diff --git src/http/ngx_http_cache.h src/http/ngx_http_cache.h index f9e96640..2910f6a1 100644 --- src/http/ngx_http_cache.h +++ src/http/ngx_http_cache.h @@ -114,6 +114,7 @@ struct ngx_http_cache_s { unsigned exists:1; unsigned temp_file:1; unsigned purged:1; + unsigned opening:1; unsigned reading:1; unsigned secondary:1; unsigned background:1; diff --git src/http/ngx_http_file_cache.c src/http/ngx_http_file_cache.c index 56866fa4..9cb110bf 100644 --- src/http/ngx_http_file_cache.c +++ src/http/ngx_http_file_cache.c @@ -18,6 +18,10 @@ static void ngx_http_file_cache_lock_wait(ngx_http_request_t *r, ngx_http_cache_t *c); static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c); +static ngx_int_t ngx_http_file_cache_aio_open(ngx_http_request_t *r, + ngx_http_cache_t *c, ngx_int_t *rv); +static ngx_int_t ngx_http_file_cache_do_aio_open(ngx_http_request_t *r, + ngx_http_cache_t *c, ngx_open_file_info_t *of, ngx_int_t *rv); static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c); #if (NGX_HAVE_FILE_AIO) @@ -268,9 +272,7 @@ ngx_http_file_cache_open(ngx_http_request_t *r) ngx_uint_t test; ngx_http_cache_t *c; ngx_pool_cleanup_t *cln; - ngx_open_file_info_t of; ngx_http_file_cache_t *cache; - ngx_http_core_loc_conf_t *clcf; c = r->cache; @@ -278,6 +280,10 @@ ngx_http_file_cache_open(ngx_http_request_t *r) return NGX_AGAIN; } + if (c->opening) { + return ngx_http_file_cache_aio_open(r, c, &rv); + } + if (c->reading) { return ngx_http_file_cache_read(r, c); } @@ -343,23 +349,33 @@ ngx_http_file_cache_open(ngx_http_request_t *r) goto done; } - clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + return ngx_http_file_cache_aio_open(r, c, &rv); - ngx_memzero(&of, sizeof(ngx_open_file_info_t)); +done: - of.uniq = c->uniq; - of.valid = clcf->open_file_cache_valid; - of.min_uses = clcf->open_file_cache_min_uses; - of.events = clcf->open_file_cache_events; - of.directio = NGX_OPEN_FILE_DIRECTIO_OFF; - of.read_ahead = clcf->read_ahead; + if (rv == NGX_DECLINED) { + return ngx_http_file_cache_lock(r, c); + } - if (ngx_open_cached_file(clcf->open_file_cache, &c->file.name, &of, r->pool) - != NGX_OK) - { - switch (of.err) { + return rv; +} + +static ngx_int_t +ngx_http_file_cache_aio_open(ngx_http_request_t *r, ngx_http_cache_t *c, + ngx_int_t *rv) +{ + ngx_int_t rc; + ngx_open_file_info_t of; + ngx_http_file_cache_t *cache = c->file_cache; - case 0: + rc = ngx_http_file_cache_do_aio_open(r, c, &of, rv); + if (rc == NGX_AGAIN) { + return rc; + } + + if (rc != NGX_OK) { + switch (of.err) { + case NGX_OK: return NGX_ERROR; case NGX_ENOENT: @@ -391,14 +407,13 @@ ngx_http_file_cache_open(ngx_http_request_t *r) done: - if (rv == NGX_DECLINED) { + if (*rv == NGX_DECLINED) { return ngx_http_file_cache_lock(r, c); } - return rv; + return *rv; } - static ngx_int_t ngx_http_file_cache_lock(ngx_http_request_t *r, ngx_http_cache_t *c) { @@ -663,6 +678,127 @@ ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c) } +#if (NGX_THREADS) +typedef struct { + ngx_str_t name; + ngx_pool_t *pool; + ngx_open_file_info_t of; + ngx_int_t rv; +} ngx_thread_file_open_ctx_t; + +static void +ngx_http_file_cache_thread_open_handler(void *data, ngx_log_t *log) +{ + ngx_thread_file_open_ctx_t *ctx = data; + ngx_pool_t *pool = ctx->pool; + ngx_open_file_info_t *of = &ctx->of; + ngx_pool_cleanup_t *cln; + ngx_pool_cleanup_file_t *clnf; + ngx_int_t rc; + + ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "thread read handler"); + + cln = ngx_pool_cleanup_add(pool, sizeof(ngx_pool_cleanup_file_t)); + if (cln == NULL) { + ctx->of.err = NGX_ERROR; + return; + } + + of->fd = NGX_INVALID_FILE; + + rc = ngx_open_and_stat_file(&ctx->name, of, pool->log); + if (rc == NGX_OK && !of->is_dir) { + cln->handler = ngx_pool_cleanup_file; + clnf = cln->data; + + clnf->fd = of->fd; + clnf->name = ctx->name.data; + clnf->log = pool->log; + } +} + + +static ngx_int_t +ngx_http_file_cache_thread_open(ngx_http_cache_t *c, ngx_open_file_info_t *of, + ngx_int_t *rv, ngx_pool_t *pool) +{ + ngx_thread_task_t *task; + ngx_thread_file_open_ctx_t *ctx; + ngx_file_t *file = &c->file; + + task = file->thread_task; + + if (task == NULL) { + task = ngx_thread_task_alloc(pool, sizeof(ngx_thread_file_open_ctx_t)); + if (task == NULL) { + return NGX_ERROR; + } + + file->thread_task = task; + } + + ctx = task->ctx; + + if (task->event.complete) { + task->event.complete = 0; + + *of = ctx->of; + *rv = ctx->rv; + return of->err; + } + + task->handler = ngx_http_file_cache_thread_open_handler; + + ctx->pool = pool; + ctx->name = file->name; + ctx->of = *of; + ctx->rv = *rv; + + if (file->thread_handler(task, file) != NGX_OK) { + return NGX_ERROR; + } + + return NGX_AGAIN; +} +#endif + +static ngx_int_t +ngx_http_file_cache_do_aio_open(ngx_http_request_t *r, ngx_http_cache_t *c, + ngx_open_file_info_t *of, ngx_int_t *rv) +{ + ngx_http_core_loc_conf_t *clcf; + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + + ngx_memzero(of, sizeof(ngx_open_file_info_t)); + + of->uniq = c->uniq; + of->valid = clcf->open_file_cache_valid; + of->min_uses = clcf->open_file_cache_min_uses; + of->events = clcf->open_file_cache_events; + of->directio = NGX_OPEN_FILE_DIRECTIO_OFF; + of->read_ahead = clcf->read_ahead; + +#if (NGX_THREADS) + if (clcf->aio == NGX_HTTP_AIO_THREADS) { + ngx_int_t rc; + + c->file.thread_task = c->thread_task; + c->file.thread_handler = ngx_http_cache_thread_handler; + c->file.thread_ctx = r; + + rc = ngx_http_file_cache_thread_open(c, of, rv, r->pool); + + c->thread_task = c->file.thread_task; + c->opening = (rc == NGX_AGAIN); + + return rc; + } +#endif + + return ngx_open_cached_file(clcf->open_file_cache, &c->file.name, of, r->pool); +} + static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c) {

1 year

Core: close pid file while writing it failed.

by Jim T

Hello! As far as I understand it, `ngx_create_pidfile` is a function that works independently. There is no action to close the pid file externally, so we need to close the pid file when the writing it failed. There are also reports here https://github.com/nginx/nginx/pull/52. # HG changeset patch # User Jinhua Tan <312841925(a)qq.com> # Date 1590068494 -28800 # Thu May 21 21:41:34 2020 +0800 # Node ID 6084ea4d9a4d2ae32f3fc4e2e3b9032ab0b71e30 # Parent 3242f98298975e556a7e87130611ce84799fe935 Core: close pid file while writing it failed. diff -r 3242f9829897 -r 6084ea4d9a4d src/core/ngx_cycle.c --- a/src/core/ngx_cycle.c Wed May 20 12:24:05 2020 +0800 +++ b/src/core/ngx_cycle.c Thu May 21 21:41:34 2020 +0800 @@ -1036,6 +1036,12 @@ len = ngx_snprintf(pid, NGX_INT64_LEN + 2, "%P%N", ngx_pid) - pid; if (ngx_write_file(&file, pid, len, 0) == NGX_ERROR) { + + if (ngx_close_file(file.fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", file.name.data); + } + return NGX_ERROR; } } Thank you!

2 years

[PATCH] introduce new variable fs_watermark for proxy_cache_path

by Adam Bambuch

# HG changeset patch # User Adam Bambuch <adam.bambuch2(a)gmail.com> # Date 1588808163 -7200 # Thu May 07 01:36:03 2020 +0200 # Node ID 8d054b64f07457cad2b74376d5f88162c887ba35 # Parent 716eddd74bc2831537f5b3f7ecd16ad3e516d043 introduce new variable fs_watermark for proxy_cache_path This configuration parameter should help with better disk usage, especially in environments, where nginx doesn't have a separate partition for caching and disk space is shared between multiple apps. It could also help in environments, where cache contains many small files, and cache loader can't load all the files in a reasonable time and therefore max_size is exceeded for many hours after nginx restart. diff -r 716eddd74bc2 -r 8d054b64f074 src/http/ngx_http_cache.h --- a/src/http/ngx_http_cache.h Thu Apr 23 15:10:26 2020 +0300 +++ b/src/http/ngx_http_cache.h Thu May 07 01:36:03 2020 +0200 @@ -160,6 +160,7 @@ ngx_path_t *path; + off_t fs_watermark; off_t max_size; size_t bsize; diff -r 716eddd74bc2 -r 8d054b64f074 src/http/ngx_http_file_cache.c --- a/src/http/ngx_http_file_cache.c Thu Apr 23 15:10:26 2020 +0300 +++ b/src/http/ngx_http_file_cache.c Thu May 07 01:36:03 2020 +0200 @@ -1959,7 +1959,7 @@ { ngx_http_file_cache_t *cache = data; - off_t size; + off_t fs_available, size; time_t wait; ngx_msec_t elapsed, next; ngx_uint_t count, watermark; @@ -1983,11 +1983,22 @@ ngx_shmtx_unlock(&cache->shpool->mutex); - ngx_log_debug3(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, - "http file cache size: %O c:%ui w:%i", - size, count, (ngx_int_t) watermark); - - if (size < cache->max_size && count < watermark) { + fs_available = NGX_MAX_OFF_T_VALUE; + if (cache->fs_watermark) { + fs_available = ngx_fs_available(cache->path->name.data); + if (fs_available == NGX_ERROR) { + ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno, + ngx_fs_available_n " \"%s\" failed", cache->path->name.data); + } + } + + ngx_log_debug4(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, + "http file cache size: %O c:%ui w:%i fa:%O", + size, count, (ngx_int_t) watermark, fs_available); + + if (size < cache->max_size && + count < watermark && + fs_available >= cache->fs_watermark) { break; } @@ -2304,7 +2315,7 @@ { char *confp = conf; - off_t max_size; + off_t fs_watermark, max_size; u_char *last, *p; time_t inactive; ssize_t size; @@ -2340,6 +2351,7 @@ name.len = 0; size = 0; + fs_watermark = 0; max_size = NGX_MAX_OFF_T_VALUE; value = cf->args->elts; @@ -2461,6 +2473,26 @@ continue; } + #if (NGX_WIN32 || NGX_HAVE_STATFS || NGX_HAVE_STATVFS) + + if (ngx_strncmp(value[i].data, "fs_watermark=", 13) == 0) { + + s.len = value[i].len - 13; + s.data = value[i].data + 13; + + fs_watermark = ngx_parse_offset(&s); + if (fs_watermark < 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid fs_watermark value \"%V\"", + &value[i]); + return NGX_CONF_ERROR; + } + + continue; + } + + #endif + if (ngx_strncmp(value[i].data, "max_size=", 9) == 0) { s.len = value[i].len - 9; @@ -2606,6 +2638,7 @@ cache->use_temp_path = use_temp_path; cache->inactive = inactive; + cache->fs_watermark = fs_watermark; cache->max_size = max_size; caches = (ngx_array_t *) (confp + cmd->offset); diff -r 716eddd74bc2 -r 8d054b64f074 src/os/unix/ngx_files.c --- a/src/os/unix/ngx_files.c Thu Apr 23 15:10:26 2020 +0300 +++ b/src/os/unix/ngx_files.c Thu May 07 01:36:03 2020 +0200 @@ -878,6 +878,18 @@ return (size_t) fs.f_bsize; } +off_t +ngx_fs_available(u_char *name) +{ + struct statfs fs; + + if (statfs((char *) name, &fs) == -1) { + return NGX_ERROR; + } + + return (ssize_t) (fs.f_bavail * fs.f_bsize); +} + #elif (NGX_HAVE_STATVFS) size_t @@ -896,6 +908,18 @@ return (size_t) fs.f_frsize; } +off_t +ngx_fs_available(u_char *name) +{ + struct statvfs fs; + + if (statvfs((char *) name, &fs) == -1) { + return NGX_ERROR; + } + + return (ssize_t) (fs.f_bavail * fs.f_bsize); +} + #else size_t @@ -904,4 +928,10 @@ return 512; } +off_t +ngx_fs_available(u_char *name) +{ + return NGX_ERROR; +} + #endif diff -r 716eddd74bc2 -r 8d054b64f074 src/os/unix/ngx_files.h --- a/src/os/unix/ngx_files.h Thu Apr 23 15:10:26 2020 +0300 +++ b/src/os/unix/ngx_files.h Thu May 07 01:36:03 2020 +0200 @@ -347,6 +347,23 @@ size_t ngx_fs_bsize(u_char *name); +off_t ngx_fs_available(u_char *name); + + +#if (NGX_HAVE_STATFS) + +#define ngx_fs_available_n "statfs()" + +#elif (NGX_HAVE_STATVFS) + +#define ngx_fs_available_n "statvfs()" + +#else + +#define ngx_fs_available_n "ngx_fs_available()" + +#endif + #if (NGX_HAVE_OPENAT) diff -r 716eddd74bc2 -r 8d054b64f074 src/os/win32/ngx_files.c --- a/src/os/win32/ngx_files.c Thu Apr 23 15:10:26 2020 +0300 +++ b/src/os/win32/ngx_files.c Thu May 07 01:36:03 2020 +0200 @@ -657,6 +657,18 @@ return sc * bs; } +off_t +ngx_fs_available(u_char *name) +{ + ULARGE_INTEGER navail, ntotal, nfree; + + if (GetDiskFreeSpaceEx((const char *) name, &navail, &ntotal, &nfree) == 0) { + return NGX_ERROR; + } + + return (off_t) navail.QuadPart; +} + static ngx_int_t ngx_win32_check_filename(u_char *name, u_short *u, size_t len) diff -r 716eddd74bc2 -r 8d054b64f074 src/os/win32/ngx_files.h --- a/src/os/win32/ngx_files.h Thu Apr 23 15:10:26 2020 +0300 +++ b/src/os/win32/ngx_files.h Thu May 07 01:36:03 2020 +0200 @@ -260,6 +260,9 @@ size_t ngx_fs_bsize(u_char *name); +off_t ngx_fs_available(u_char *name); +#define ngx_fs_available_n "GetDiskFreeSpaceEx" + #define ngx_stdout GetStdHandle(STD_OUTPUT_HANDLE) #define ngx_stderr GetStdHandle(STD_ERROR_HANDLE)

2 years, 2 months

[PATCH] Ensured SIGQUIT deletes listening UNIX socket files.

by Thibault Charbonnier

# HG changeset patch # User Thibault Charbonnier <thibaultcha(a)me.com> # Date 1582764433 28800 # Wed Feb 26 16:47:13 2020 -0800 # Node ID 55ea1a9197a6f28d4da00909e5ea8585f6a08239 # Parent 4f18393a1d51bce6103ea2f1b2587900f349ba3d Ensured SIGQUIT deletes listening UNIX socket files. Prior to this patch, the SIGQUIT signal handling (graceful shutdown) did not remove UNIX socket files since ngx_master_process_cycle reimplemented listening socket closings in lieu of using ngx_close_listening_sockets. Since ngx_master_process_exit will call the aforementioned ngx_close_listening_sockets, we can remove the custom implementation and now expect listening sockets to be closed properly by ngx_close_listening_sockets instead. This fixes the trac issue #753 (https://trac.nginx.org/nginx/ticket/753). diff -r 4f18393a1d51 -r 55ea1a9197a6 src/os/unix/ngx_process_cycle.c --- a/src/os/unix/ngx_process_cycle.c Thu Feb 20 16:51:07 2020 +0300 +++ b/src/os/unix/ngx_process_cycle.c Wed Feb 26 16:47:13 2020 -0800 @@ -77,12 +77,11 @@ u_char *p; size_t size; ngx_int_t i; - ngx_uint_t n, sigio; + ngx_uint_t sigio; sigset_t set; struct itimerval itv; ngx_uint_t live; ngx_msec_t delay; - ngx_listening_t *ls; ngx_core_conf_t *ccf; sigemptyset(&set); @@ -205,16 +204,6 @@ ngx_signal_worker_processes(cycle, ngx_signal_value(NGX_SHUTDOWN_SIGNAL)); - ls = cycle->listening.elts; - for (n = 0; n < cycle->listening.nelts; n++) { - if (ngx_close_socket(ls[n].fd) == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, - ngx_close_socket_n " %V failed", - &ls[n].addr_text); - } - } - cycle->listening.nelts = 0; - continue; }

2 years, 2 months

[njs] Added AST serialization.

by Dmitry Volyntsev

details: https://hg.nginx.org/njs/rev/d255e73aed3b branches: changeset: 1408:d255e73aed3b user: Dmitry Volyntsev <xeioex(a)nginx.com> date: Fri May 29 19:29:59 2020 +0000 description: Added AST serialization. diffstat: src/njs.h | 2 + src/njs_parser.c | 256 +++++++++++++++++++++++++++++++++++++++++++++++ src/njs_parser.h | 2 + src/njs_shell.c | 7 + src/njs_vm.c | 21 +++- test/njs_expect_test.exp | 4 + 6 files changed, 291 insertions(+), 1 deletions(-) diffs (391 lines): diff -r 86f55a7dc4a4 -r d255e73aed3b src/njs.h --- a/src/njs.h Fri May 29 20:00:32 2020 +0300 +++ b/src/njs.h Fri May 29 19:29:59 2020 +0000 @@ -206,6 +206,7 @@ typedef struct { * unsafe - enables unsafe language features: * - Function constructors. * module - ES6 "module" mode. Script mode is default. + * ast - print AST. */ uint8_t trailer; /* 1 bit */ @@ -217,6 +218,7 @@ typedef struct { uint8_t sandbox; /* 1 bit */ uint8_t unsafe; /* 1 bit */ uint8_t module; /* 1 bit */ + uint8_t ast; /* 1 bit */ } njs_vm_opt_t; diff -r 86f55a7dc4a4 -r d255e73aed3b src/njs_parser.c --- a/src/njs_parser.c Fri May 29 20:00:32 2020 +0300 +++ b/src/njs_parser.c Fri May 29 19:29:59 2020 +0000 @@ -449,6 +449,11 @@ static njs_token_type_t njs_parser_escap static njs_int_t njs_parser_escape_string_calc_length(njs_parser_t *parser, njs_lexer_token_t *token, size_t *out_size, size_t *out_length); +static void njs_parser_serialize_tree(njs_chb_t *chain, + njs_parser_node_t *node, njs_int_t *ret, size_t indent); +static njs_int_t njs_parser_serialize_node(njs_chb_t *chain, + njs_parser_node_t *node); + #define njs_parser_chain_top(parser) \ ((parser)->scope->top) @@ -8269,3 +8274,254 @@ njs_parser_node_error(njs_vm_t *vm, njs_ njs_parser_scope_error(vm, node->scope, type, node->token_line, fmt, args); va_end(args); } + + +njs_int_t +njs_parser_serialize_ast(njs_parser_node_t *node, njs_chb_t *chain) +{ + njs_int_t ret; + + ret = NJS_OK; + + njs_parser_serialize_tree(chain, node, &ret, 0); + njs_chb_append_literal(chain, "\n"); + + return ret; +} + + +njs_inline void +njs_parser_serialize_indent(njs_chb_t *chain, size_t indent) +{ + size_t i; + + for (i = 0; i < indent; i++) { + njs_chb_append_literal(chain, " "); + } +} + + +static void +njs_parser_serialize_tree(njs_chb_t *chain, njs_parser_node_t *node, + njs_int_t *ret, size_t indent) +{ + njs_str_t str; + + njs_chb_append_literal(chain, "{\"name\": \""); + + *ret |= njs_parser_serialize_node(chain, node); + + njs_chb_append_literal(chain, "\""); + + switch (node->token_type) { + case NJS_TOKEN_NUMBER: + case NJS_TOKEN_STRING: + case NJS_TOKEN_NAME: + case NJS_TOKEN_FUNCTION_CALL: + njs_chb_append_literal(chain, ",\n"); + njs_parser_serialize_indent(chain, indent); + njs_chb_sprintf(chain, 32, " \"index\": \"%p\"", node->index); + + switch (node->token_type) { + case NJS_TOKEN_NUMBER: + case NJS_TOKEN_STRING: + njs_chb_append_literal(chain, ",\n"); + njs_parser_serialize_indent(chain, indent); + + if (node->token_type == NJS_TOKEN_NUMBER) { + njs_chb_sprintf(chain, 32, " \"value\": %f\n", + njs_number(&node->u.value)); + + } else { + njs_string_get(&node->u.value, &str); + njs_chb_sprintf(chain, 32, " \"value\": \"%V\"\n", &str); + } + + break; + + default: + break; + } + + break; + + default: + break; + } + + if (node->left != NULL) { + njs_chb_append_literal(chain, ",\n"); + njs_parser_serialize_indent(chain, indent); + njs_chb_append_literal(chain, " \"left\": "); + + njs_parser_serialize_tree(chain, node->left, ret, indent + 1); + } + + if (node->right != NULL) { + njs_chb_append_literal(chain, ",\n"); + njs_parser_serialize_indent(chain, indent); + njs_chb_append_literal(chain, " \"right\": "); + + njs_parser_serialize_tree(chain, node->right, ret, indent + 1); + } + + njs_chb_append_literal(chain, "}"); +} + + +static njs_int_t +njs_parser_serialize_node(njs_chb_t *chain, njs_parser_node_t *node) +{ + const char *name; + +#define njs_token_serialize(token) \ + case token: \ + name = &njs_stringify(token)[njs_length("NJS_TOKEN_")]; \ + njs_chb_append(chain, name, njs_strlen(name)); \ + break + + switch (node->token_type) { + njs_token_serialize(NJS_TOKEN_END); + /* FIXME: NJS_TOKEN_ILLEGAL should not be present in AST */ + njs_token_serialize(NJS_TOKEN_ILLEGAL); + njs_token_serialize(NJS_TOKEN_COMMA); + njs_token_serialize(NJS_TOKEN_CONDITIONAL); + njs_token_serialize(NJS_TOKEN_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_ADDITION_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_SUBSTRACTION_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_MULTIPLICATION_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_EXPONENTIATION_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_DIVISION_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_REMAINDER_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_LEFT_SHIFT_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_RIGHT_SHIFT_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_UNSIGNED_RIGHT_SHIFT_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_BITWISE_OR_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_BITWISE_XOR_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_BITWISE_AND_ASSIGNMENT); + njs_token_serialize(NJS_TOKEN_EQUAL); + njs_token_serialize(NJS_TOKEN_NOT_EQUAL); + njs_token_serialize(NJS_TOKEN_STRICT_EQUAL); + njs_token_serialize(NJS_TOKEN_STRICT_NOT_EQUAL); + njs_token_serialize(NJS_TOKEN_ADDITION); + njs_token_serialize(NJS_TOKEN_UNARY_PLUS); + njs_token_serialize(NJS_TOKEN_INCREMENT); + njs_token_serialize(NJS_TOKEN_POST_INCREMENT); + njs_token_serialize(NJS_TOKEN_SUBSTRACTION); + njs_token_serialize(NJS_TOKEN_UNARY_NEGATION); + njs_token_serialize(NJS_TOKEN_DECREMENT); + njs_token_serialize(NJS_TOKEN_POST_DECREMENT); + njs_token_serialize(NJS_TOKEN_MULTIPLICATION); + njs_token_serialize(NJS_TOKEN_EXPONENTIATION); + njs_token_serialize(NJS_TOKEN_DIVISION); + njs_token_serialize(NJS_TOKEN_REMAINDER); + njs_token_serialize(NJS_TOKEN_LESS); + njs_token_serialize(NJS_TOKEN_LESS_OR_EQUAL); + njs_token_serialize(NJS_TOKEN_LEFT_SHIFT); + njs_token_serialize(NJS_TOKEN_GREATER); + njs_token_serialize(NJS_TOKEN_GREATER_OR_EQUAL); + njs_token_serialize(NJS_TOKEN_RIGHT_SHIFT); + njs_token_serialize(NJS_TOKEN_UNSIGNED_RIGHT_SHIFT); + njs_token_serialize(NJS_TOKEN_BITWISE_OR); + njs_token_serialize(NJS_TOKEN_LOGICAL_OR); + njs_token_serialize(NJS_TOKEN_BITWISE_XOR); + njs_token_serialize(NJS_TOKEN_BITWISE_AND); + njs_token_serialize(NJS_TOKEN_LOGICAL_AND); + njs_token_serialize(NJS_TOKEN_BITWISE_NOT); + njs_token_serialize(NJS_TOKEN_LOGICAL_NOT); + njs_token_serialize(NJS_TOKEN_COALESCE); + njs_token_serialize(NJS_TOKEN_IN); + njs_token_serialize(NJS_TOKEN_OF); + njs_token_serialize(NJS_TOKEN_INSTANCEOF); + njs_token_serialize(NJS_TOKEN_TYPEOF); + njs_token_serialize(NJS_TOKEN_VOID); + njs_token_serialize(NJS_TOKEN_NEW); + njs_token_serialize(NJS_TOKEN_DELETE); + njs_token_serialize(NJS_TOKEN_YIELD); + + njs_token_serialize(NJS_TOKEN_NULL); + njs_token_serialize(NJS_TOKEN_NUMBER); + njs_token_serialize(NJS_TOKEN_TRUE); + njs_token_serialize(NJS_TOKEN_FALSE); + njs_token_serialize(NJS_TOKEN_STRING); + njs_token_serialize(NJS_TOKEN_TEMPLATE_LITERAL); + njs_token_serialize(NJS_TOKEN_NAME); + njs_token_serialize(NJS_TOKEN_OBJECT); + njs_token_serialize(NJS_TOKEN_OBJECT_VALUE); + njs_token_serialize(NJS_TOKEN_ARRAY); + njs_token_serialize(NJS_TOKEN_REGEXP); + + njs_token_serialize(NJS_TOKEN_PROPERTY); + njs_token_serialize(NJS_TOKEN_PROPERTY_INIT); + njs_token_serialize(NJS_TOKEN_PROPERTY_DELETE); + njs_token_serialize(NJS_TOKEN_PROPERTY_GETTER); + njs_token_serialize(NJS_TOKEN_PROPERTY_SETTER); + + njs_token_serialize(NJS_TOKEN_PROTO_INIT); + + njs_token_serialize(NJS_TOKEN_FUNCTION); + njs_token_serialize(NJS_TOKEN_FUNCTION_EXPRESSION); + njs_token_serialize(NJS_TOKEN_FUNCTION_CALL); + njs_token_serialize(NJS_TOKEN_METHOD_CALL); + + njs_token_serialize(NJS_TOKEN_ARGUMENT); + njs_token_serialize(NJS_TOKEN_RETURN); + njs_token_serialize(NJS_TOKEN_STATEMENT); + njs_token_serialize(NJS_TOKEN_BLOCK); + njs_token_serialize(NJS_TOKEN_VAR); + njs_token_serialize(NJS_TOKEN_LET); + njs_token_serialize(NJS_TOKEN_IF); + njs_token_serialize(NJS_TOKEN_ELSE); + njs_token_serialize(NJS_TOKEN_BRANCHING); + njs_token_serialize(NJS_TOKEN_WHILE); + njs_token_serialize(NJS_TOKEN_DO); + njs_token_serialize(NJS_TOKEN_FOR); + njs_token_serialize(NJS_TOKEN_FOR_IN); + njs_token_serialize(NJS_TOKEN_BREAK); + njs_token_serialize(NJS_TOKEN_CONTINUE); + njs_token_serialize(NJS_TOKEN_SWITCH); + njs_token_serialize(NJS_TOKEN_CASE); + njs_token_serialize(NJS_TOKEN_DEFAULT); + njs_token_serialize(NJS_TOKEN_WITH); + njs_token_serialize(NJS_TOKEN_TRY); + njs_token_serialize(NJS_TOKEN_CATCH); + njs_token_serialize(NJS_TOKEN_FINALLY); + njs_token_serialize(NJS_TOKEN_THROW); + njs_token_serialize(NJS_TOKEN_THIS); + njs_token_serialize(NJS_TOKEN_GLOBAL_OBJECT); + njs_token_serialize(NJS_TOKEN_NON_LOCAL_THIS); + njs_token_serialize(NJS_TOKEN_ARGUMENTS); + njs_token_serialize(NJS_TOKEN_EVAL); + njs_token_serialize(NJS_TOKEN_IMPORT); + njs_token_serialize(NJS_TOKEN_EXPORT); + +#if 0 + + njs_token_serialize(NJS_TOKEN_TARGET); + njs_token_serialize(NJS_TOKEN_META); + njs_token_serialize(NJS_TOKEN_ASYNC); + njs_token_serialize(NJS_TOKEN_AWAIT); + njs_token_serialize(NJS_TOKEN_CONST); + njs_token_serialize(NJS_TOKEN_DEBUGGER); + njs_token_serialize(NJS_TOKEN_ENUM); + + njs_token_serialize(NJS_TOKEN_CLASS); + njs_token_serialize(NJS_TOKEN_EXTENDS); + njs_token_serialize(NJS_TOKEN_IMPLEMENTS); + njs_token_serialize(NJS_TOKEN_INTERFACE); + njs_token_serialize(NJS_TOKEN_PACKAGE); + njs_token_serialize(NJS_TOKEN_PRIVATE); + njs_token_serialize(NJS_TOKEN_PROTECTED); + njs_token_serialize(NJS_TOKEN_PUBLIC); + njs_token_serialize(NJS_TOKEN_STATIC); + njs_token_serialize(NJS_TOKEN_SUPER); + +#endif + + default: + njs_chb_sprintf(chain, 32, "#UNDEF(%d)", (int) node->token_type); + return NJS_DECLINED; + } + + return NJS_OK; +} diff -r 86f55a7dc4a4 -r d255e73aed3b src/njs_parser.h --- a/src/njs_parser.h Fri May 29 20:00:32 2020 +0300 +++ b/src/njs_parser.h Fri May 29 19:29:59 2020 +0000 @@ -128,6 +128,8 @@ void njs_parser_lexer_error(njs_parser_t void njs_parser_node_error(njs_vm_t *vm, njs_parser_node_t *node, njs_object_type_t type, const char *fmt, ...); +njs_int_t njs_parser_serialize_ast(njs_parser_node_t *node, njs_chb_t *chain); + #define njs_parser_restricted_identifier(token) \ (token == NJS_TOKEN_ARGUMENTS || token == NJS_TOKEN_EVAL) diff -r 86f55a7dc4a4 -r d255e73aed3b src/njs_shell.c --- a/src/njs_shell.c Fri May 29 20:00:32 2020 +0300 +++ b/src/njs_shell.c Fri May 29 19:29:59 2020 +0000 @@ -34,6 +34,7 @@ typedef struct { uint8_t sandbox; uint8_t safe; uint8_t version; + uint8_t ast; char *file; char *command; @@ -268,6 +269,7 @@ main(int argc, char **argv) vm_options.external = &njs_console; vm_options.argv = opts.argv; vm_options.argc = opts.argc; + vm_options.ast = opts.ast; if (opts.interactive) { ret = njs_interactive_shell(&opts, &vm_options); @@ -307,6 +309,7 @@ njs_get_options(njs_opts_t *opts, int ar "njs [options] [-c string | script.js | -] [script args]" "\n" "Options:\n" + " -a print AST.\n" " -c specify the command to execute.\n" " -d print disassembled code.\n" " -f disabled denormals mode.\n" @@ -340,6 +343,10 @@ njs_get_options(njs_opts_t *opts, int ar (void) write(STDOUT_FILENO, help, njs_length(help)); return ret; + case 'a': + opts->ast = 1; + break; + case 'c': opts->interactive = 0; diff -r 86f55a7dc4a4 -r d255e73aed3b src/njs_vm.c --- a/src/njs_vm.c Fri May 29 20:00:32 2020 +0300 +++ b/src/njs_vm.c Fri May 29 19:29:59 2020 +0000 @@ -125,6 +125,8 @@ njs_int_t njs_vm_compile(njs_vm_t *vm, u_char **start, u_char *end) { njs_int_t ret; + njs_str_t ast; + njs_chb_t chain; njs_lexer_t lexer; njs_parser_t *parser, *prev; njs_generator_t generator; @@ -202,7 +204,24 @@ njs_vm_compile(njs_vm_t *vm, u_char **st njs_disassembler(vm); } - return NJS_OK; + if (njs_slow_path(vm->options.ast)) { + njs_chb_init(&chain, vm->mem_pool); + ret = njs_parser_serialize_ast(parser->node, &chain); + if (njs_slow_path(ret == NJS_ERROR)) { + return ret; + } + + if (njs_slow_path(njs_chb_join(&chain, &ast) != NJS_OK)) { + return NJS_ERROR; + } + + njs_print(ast.start, ast.length); + + njs_chb_destroy(&chain); + njs_mp_free(vm->mem_pool, ast.start); + } + + return ret; fail: diff -r 86f55a7dc4a4 -r d255e73aed3b test/njs_expect_test.exp --- a/test/njs_expect_test.exp Fri May 29 20:00:32 2020 +0300 +++ b/test/njs_expect_test.exp Fri May 29 19:29:59 2020 +0000 @@ -791,6 +791,10 @@ njs_run {"-p" "test/module" "./test/modu njs_run {"-h"} "Options" +# ast + +njs_run {"-a" "-c" "console.log(1*2)"} "{\"name\": \"END\"" + # command njs_run {"-c" "console.log(\"a b c\")"} "a b c"

2 years, 2 months

[njs] Parser refactoring.

by Alexander Borisov

details: https://hg.nginx.org/njs/rev/86f55a7dc4a4 branches: changeset: 1407:86f55a7dc4a4 user: Alexander Borisov <alexander.borisov(a)nginx.com> date: Fri May 29 20:00:32 2020 +0300 description: Parser refactoring. diffstat: auto/sources | 2 - src/njs_function.c | 7 +- src/njs_lexer.c | 202 +- src/njs_lexer.h | 91 +- src/njs_lexer_tables.h | 491 +- src/njs_module.c | 175 +- src/njs_module.h | 3 +- src/njs_parser.c | 9622 ++++++++++++++++++++++++++++++++++-------- src/njs_parser.h | 316 +- src/njs_parser_expression.c | 1121 ----- src/njs_parser_terminal.c | 1296 ----- src/njs_regexp.c | 95 +- src/njs_regexp.h | 4 +- src/njs_variable.c | 24 +- src/njs_vm.c | 3 +- src/test/njs_unit_test.c | 57 +- test/njs_expect_test.exp | 4 +- utils/lexer_keyword.py | 129 +- 18 files changed, 8745 insertions(+), 4897 deletions(-) diffs (truncated from 14514 to 1000 lines): diff -r 1d071c0e23e8 -r 86f55a7dc4a4 auto/sources --- a/auto/sources Fri May 29 14:49:36 2020 +0300 +++ b/auto/sources Fri May 29 20:00:32 2020 +0300 @@ -49,8 +49,6 @@ NJS_LIB_SRCS=" \ src/njs_lexer.c \ src/njs_lexer_keyword.c \ src/njs_parser.c \ - src/njs_parser_terminal.c \ - src/njs_parser_expression.c \ src/njs_generator.c \ src/njs_disassembler.c \ src/njs_array_buffer.c \ diff -r 1d071c0e23e8 -r 86f55a7dc4a4 src/njs_function.c --- a/src/njs_function.c Fri May 29 14:49:36 2020 +0300 +++ b/src/njs_function.c Fri May 29 20:00:32 2020 +0300 @@ -889,7 +889,9 @@ njs_function_constructor(njs_vm_t *vm, n return ret; } - njs_chb_append_literal(&chain, ","); + if (i != (nargs - 2)) { + njs_chb_append_literal(&chain, ","); + } } njs_chb_append_literal(&chain, "){"); @@ -923,9 +925,10 @@ njs_function_constructor(njs_vm_t *vm, n return ret; } + parser->vm = vm; parser->lexer = &lexer; - ret = njs_parser(vm, parser, NULL); + ret = njs_parser(parser, NULL); if (njs_slow_path(ret != NJS_OK)) { return ret; } diff -r 1d071c0e23e8 -r 86f55a7dc4a4 src/njs_lexer.c --- a/src/njs_lexer.c Fri May 29 14:49:36 2020 +0300 +++ b/src/njs_lexer.c Fri May 29 20:00:32 2020 +0300 @@ -27,10 +27,6 @@ static void njs_lexer_multi(njs_lexer_t const njs_lexer_multi_t *multi, size_t length); static void njs_lexer_division(njs_lexer_t *lexer, njs_lexer_token_t *token); -static njs_lexer_token_t *njs_lexer_token_push(njs_vm_t *vm, - njs_lexer_t *lexer); -static njs_lexer_token_t *njs_lexer_token_pop(njs_lexer_t *lexer); - const njs_lvlhsh_proto_t njs_lexer_hash_proto njs_aligned(64) = @@ -311,102 +307,19 @@ njs_lexer_init(njs_vm_t *vm, njs_lexer_t } -njs_token_type_t -njs_lexer_token(njs_vm_t *vm, njs_lexer_t *lexer) -{ - njs_lexer_token_t *lt; - - lexer->prev_start = lexer->start; - - if (lexer->token != NULL) { - lexer->prev_type = lexer->token->type; - njs_mp_free(vm->mem_pool, lexer->token); - } - - if (njs_queue_is_empty(&lexer->preread)) { - lt = njs_lexer_token_push(vm, lexer); - if (njs_slow_path(lt == NULL)) { - return NJS_TOKEN_ERROR; - } - } - - lexer->token = njs_lexer_token_pop(lexer); - - return lexer->token->type; -} - - -njs_token_type_t -njs_lexer_peek_token(njs_vm_t *vm, njs_lexer_t *lexer, size_t offset) -{ - size_t i; - njs_queue_link_t *link; - njs_lexer_token_t *lt; - - /* GCC and Clang complain about uninitialized lt. */ - lt = NULL; - - link = njs_queue_first(&lexer->preread); - - for (i = 0; i <= offset; i++) { - - if (link != njs_queue_tail(&lexer->preread)) { - - lt = njs_queue_link_data(link, njs_lexer_token_t, link); - - /* NJS_TOKEN_DIVISION stands for regexp literal. */ - - if (lt->type == NJS_TOKEN_DIVISION || lt->type == NJS_TOKEN_END) { - break; - } - - link = njs_queue_next(link); - - } else { - - lt = njs_lexer_token_push(vm, lexer); - - if (njs_slow_path(lt == NULL)) { - return NJS_TOKEN_ERROR; - } - } - } - - return lt->type; -} - - -njs_int_t -njs_lexer_rollback(njs_vm_t *vm, njs_lexer_t *lexer) -{ - njs_lexer_token_t *lt; - - lt = njs_mp_zalloc(vm->mem_pool, sizeof(njs_lexer_token_t)); - if (njs_slow_path(lt == NULL)) { - return NJS_ERROR; - } - - *lt = *lexer->token; - - njs_queue_insert_head(&lexer->preread, &lt->link); - - return NJS_OK; -} - - -static njs_lexer_token_t * -njs_lexer_token_push(njs_vm_t *vm, njs_lexer_t *lexer) +njs_inline njs_lexer_token_t * +njs_lexer_next_token(njs_lexer_t *lexer) { njs_int_t ret; njs_lexer_token_t *token; - token = njs_mp_zalloc(vm->mem_pool, sizeof(njs_lexer_token_t)); + token = njs_mp_zalloc(lexer->mem_pool, sizeof(njs_lexer_token_t)); if (njs_slow_path(token == NULL)) { return NULL; } do { - ret = njs_lexer_next_token(lexer, token); + ret = njs_lexer_make_token(lexer, token); if (njs_slow_path(ret != NJS_OK)) { return NULL; } @@ -419,20 +332,109 @@ njs_lexer_token_push(njs_vm_t *vm, njs_l } -static njs_lexer_token_t * -njs_lexer_token_pop(njs_lexer_t *lexer) +njs_lexer_token_t * +njs_lexer_token(njs_lexer_t *lexer, njs_bool_t with_end_line) { - njs_queue_link_t *lnk; + njs_queue_link_t *lnk; + njs_lexer_token_t *token; lnk = njs_queue_first(&lexer->preread); - njs_queue_remove(lnk); + + while (lnk != njs_queue_head(&lexer->preread)) { + token = njs_queue_link_data(lnk, njs_lexer_token_t, link); + + if (!with_end_line && token->type == NJS_TOKEN_LINE_END) { + lexer->prev_type = token->type; + + lnk = njs_queue_next(&token->link); + continue; + } + + return token; + } + + do { + token = njs_lexer_next_token(lexer); + if (token == NULL) { + return NULL; + } + + if (!with_end_line && token->type == NJS_TOKEN_LINE_END) { + lexer->prev_type = token->type; + continue; + } + + break; + + } while (1); + + return token; +} + + +njs_lexer_token_t * +njs_lexer_peek_token(njs_lexer_t *lexer, njs_lexer_token_t *current, + njs_bool_t with_end_line) +{ + njs_queue_link_t *lnk; + njs_lexer_token_t *token; + + lnk = njs_queue_next(&current->link); + + while (lnk != njs_queue_head(&lexer->preread)) { + token = njs_queue_link_data(lnk, njs_lexer_token_t, link); - return njs_queue_link_data(lnk, njs_lexer_token_t, link); + if (!with_end_line && token->type == NJS_TOKEN_LINE_END) { + lnk = njs_queue_next(&token->link); + continue; + } + + return token; + } + + do { + token = njs_lexer_next_token(lexer); + if (token == NULL) { + return NULL; + } + + if (!with_end_line && token->type == NJS_TOKEN_LINE_END) { + continue; + } + + break; + + } while (1); + + return token; +} + + +void +njs_lexer_consume_token(njs_lexer_t *lexer, unsigned length) +{ + njs_queue_link_t *lnk; + njs_lexer_token_t *token; + + while (length > 0) { + lnk = njs_queue_first(&lexer->preread); + token = njs_queue_link_data(lnk, njs_lexer_token_t, link); + + lexer->prev_type = token->type; + + if (token->type != NJS_TOKEN_LINE_END) { + length--; + } + + njs_queue_remove(lnk); + + njs_mp_free(lexer->mem_pool, token); + } } njs_int_t -njs_lexer_next_token(njs_lexer_t *lexer, njs_lexer_token_t *token) +njs_lexer_make_token(njs_lexer_t *lexer, njs_lexer_token_t *token) { u_char c, *p; @@ -446,7 +448,6 @@ njs_lexer_next_token(njs_lexer_t *lexer, } } - lexer->keyword = 0; token->type = njs_tokens[c]; switch (token->type) { @@ -565,6 +566,7 @@ njs_lexer_next_token(njs_lexer_t *lexer, /* Fall through. */ default: + token->line = lexer->line; token->text.start = lexer->start - 1; token->text.length = lexer->start - token->text.start; @@ -694,12 +696,14 @@ njs_lexer_word(njs_lexer_t *lexer, njs_l } token->type = NJS_TOKEN_NAME; + token->keyword_type = NJS_KEYWORD_TYPE_UNDEF; } else { entry = &key_entry->value->entry; token->type = key_entry->value->type; - lexer->keyword = 1; + token->keyword_type = NJS_KEYWORD_TYPE_KEYWORD; + token->keyword_type |= key_entry->value->reserved; } token->unique_id = (uintptr_t) entry; diff -r 1d071c0e23e8 -r 86f55a7dc4a4 src/njs_lexer.h --- a/src/njs_lexer.h Fri May 29 14:49:36 2020 +0300 +++ b/src/njs_lexer.h Fri May 29 20:00:32 2020 +0300 @@ -99,6 +99,7 @@ typedef enum { NJS_TOKEN_COALESCE, NJS_TOKEN_IN, + NJS_TOKEN_OF, NJS_TOKEN_INSTANCEOF, NJS_TOKEN_TYPEOF, NJS_TOKEN_VOID, @@ -178,7 +179,14 @@ typedef enum { NJS_TOKEN_IMPORT, NJS_TOKEN_EXPORT, + NJS_TOKEN_TARGET, + + NJS_TOKEN_FROM, + + NJS_TOKEN_META, + NJS_TOKEN_AWAIT, + NJS_TOKEN_ASYNC, NJS_TOKEN_CLASS, NJS_TOKEN_CONST, NJS_TOKEN_DEBUGGER, @@ -198,6 +206,13 @@ typedef enum { } njs_token_type_t; +typedef enum { + NJS_KEYWORD_TYPE_UNDEF = 0, + NJS_KEYWORD_TYPE_RESERVED = 1, + NJS_KEYWORD_TYPE_KEYWORD = 2 +} njs_keyword_type_t; + + typedef struct { njs_str_t name; } njs_lexer_entry_t; @@ -206,6 +221,7 @@ typedef struct { typedef struct { njs_lexer_entry_t entry; njs_token_type_t type; + njs_bool_t reserved; } njs_keyword_t; @@ -220,6 +236,7 @@ typedef struct { typedef struct { njs_token_type_t type:16; + njs_keyword_type_t keyword_type; uint32_t line; uintptr_t unique_id; njs_str_t text; @@ -231,10 +248,10 @@ typedef struct { typedef struct { njs_lexer_token_t *token; njs_queue_t preread; /* of njs_lexer_token_t */ - uint8_t keyword; u_char *prev_start; njs_token_type_t prev_type:16; + njs_token_type_t last_type:16; uint32_t line; njs_str_t file; @@ -251,12 +268,12 @@ typedef struct { njs_int_t njs_lexer_init(njs_vm_t *vm, njs_lexer_t *lexer, njs_str_t *file, u_char *start, u_char *end); -njs_token_type_t njs_lexer_token(njs_vm_t *vm, njs_lexer_t *lexer); -njs_token_type_t njs_lexer_peek_token(njs_vm_t *vm, njs_lexer_t *lexer, - size_t offset); -njs_int_t njs_lexer_rollback(njs_vm_t *vm, njs_lexer_t *lexer); - -njs_int_t njs_lexer_next_token(njs_lexer_t *lexer, njs_lexer_token_t *token); +njs_lexer_token_t *njs_lexer_token(njs_lexer_t *lexer, + njs_bool_t with_end_line); +njs_lexer_token_t *njs_lexer_peek_token(njs_lexer_t *lexer, + njs_lexer_token_t *current, njs_bool_t with_end_line); +void njs_lexer_consume_token(njs_lexer_t *lexer, unsigned length); +njs_int_t njs_lexer_make_token(njs_lexer_t *lexer, njs_lexer_token_t *token); const njs_lexer_keyword_entry_t *njs_lexer_keyword(const u_char *key, size_t length); @@ -270,6 +287,66 @@ njs_lexer_entry(uintptr_t unique_id) } +njs_inline njs_bool_t +njs_lexer_token_is_keyword(njs_lexer_token_t *token) +{ + return token->keyword_type & NJS_KEYWORD_TYPE_KEYWORD; +} + + +njs_inline njs_bool_t +njs_lexer_token_is_reserved(njs_lexer_token_t *token) +{ + return token->keyword_type & NJS_KEYWORD_TYPE_RESERVED; +} + + +njs_inline njs_bool_t +njs_lexer_token_is_name(njs_lexer_token_t *token) +{ + return token->type == NJS_TOKEN_NAME + || (!njs_lexer_token_is_reserved(token) + && njs_lexer_token_is_keyword(token)); +} + + +njs_inline njs_bool_t +njs_lexer_token_is_identifier_name(njs_lexer_token_t *token) +{ + return token->type == NJS_TOKEN_NAME || njs_lexer_token_is_keyword(token); +} + + +njs_inline njs_bool_t +njs_lexer_token_is_binding_identifier(njs_lexer_token_t *token) +{ + switch (token->type) { + case NJS_TOKEN_NAME: + case NJS_TOKEN_YIELD: + case NJS_TOKEN_AWAIT: + return 1; + + default: + return (!njs_lexer_token_is_reserved(token) + && njs_lexer_token_is_keyword(token)); + }; +} + + +njs_inline njs_bool_t +njs_lexer_token_is_label_identifier(njs_lexer_token_t *token) +{ + return njs_lexer_token_is_binding_identifier(token); +} + + +njs_inline njs_bool_t +njs_lexer_token_is_identifier_reference(njs_lexer_token_t *token) +{ + return njs_lexer_token_is_binding_identifier(token); +} + + extern const njs_lvlhsh_proto_t njs_lexer_hash_proto; diff -r 1d071c0e23e8 -r 86f55a7dc4a4 src/njs_lexer_tables.h --- a/src/njs_lexer_tables.h Fri May 29 14:49:36 2020 +0300 +++ b/src/njs_lexer_tables.h Fri May 29 20:00:32 2020 +0300 @@ -10,135 +10,428 @@ #define _NJS_LEXER_TABLES_H_INCLUDED_ -static const njs_keyword_t njs_lexer_kws[48] = +static const njs_keyword_t njs_lexer_kws[53] = { - { .entry = { njs_str("null") }, .type = NJS_TOKEN_NULL }, - { .entry = { njs_str("false") }, .type = NJS_TOKEN_FALSE }, - { .entry = { njs_str("true") }, .type = NJS_TOKEN_TRUE }, - { .entry = { njs_str("in") }, .type = NJS_TOKEN_IN }, - { .entry = { njs_str("typeof") }, .type = NJS_TOKEN_TYPEOF }, - { .entry = { njs_str("instanceof") }, .type = NJS_TOKEN_INSTANCEOF }, - { .entry = { njs_str("void") }, .type = NJS_TOKEN_VOID }, - { .entry = { njs_str("new") }, .type = NJS_TOKEN_NEW }, - { .entry = { njs_str("delete") }, .type = NJS_TOKEN_DELETE }, - { .entry = { njs_str("yield") }, .type = NJS_TOKEN_YIELD }, - { .entry = { njs_str("var") }, .type = NJS_TOKEN_VAR }, - { .entry = { njs_str("if") }, .type = NJS_TOKEN_IF }, - { .entry = { njs_str("else") }, .type = NJS_TOKEN_ELSE }, - { .entry = { njs_str("while") }, .type = NJS_TOKEN_WHILE }, - { .entry = { njs_str("do") }, .type = NJS_TOKEN_DO }, - { .entry = { njs_str("for") }, .type = NJS_TOKEN_FOR }, - { .entry = { njs_str("break") }, .type = NJS_TOKEN_BREAK }, - { .entry = { njs_str("continue") }, .type = NJS_TOKEN_CONTINUE }, - { .entry = { njs_str("switch") }, .type = NJS_TOKEN_SWITCH }, - { .entry = { njs_str("case") }, .type = NJS_TOKEN_CASE }, - { .entry = { njs_str("default") }, .type = NJS_TOKEN_DEFAULT }, - { .entry = { njs_str("function") }, .type = NJS_TOKEN_FUNCTION }, - { .entry = { njs_str("return") }, .type = NJS_TOKEN_RETURN }, - { .entry = { njs_str("with") }, .type = NJS_TOKEN_WITH }, - { .entry = { njs_str("try") }, .type = NJS_TOKEN_TRY }, - { .entry = { njs_str("catch") }, .type = NJS_TOKEN_CATCH }, - { .entry = { njs_str("finally") }, .type = NJS_TOKEN_FINALLY }, - { .entry = { njs_str("throw") }, .type = NJS_TOKEN_THROW }, - { .entry = { njs_str("import") }, .type = NJS_TOKEN_IMPORT }, - { .entry = { njs_str("export") }, .type = NJS_TOKEN_EXPORT }, - { .entry = { njs_str("this") }, .type = NJS_TOKEN_THIS }, - { .entry = { njs_str("arguments") }, .type = NJS_TOKEN_ARGUMENTS }, - { .entry = { njs_str("eval") }, .type = NJS_TOKEN_EVAL }, - { .entry = { njs_str("await") }, .type = NJS_TOKEN_AWAIT }, - { .entry = { njs_str("class") }, .type = NJS_TOKEN_CLASS }, - { .entry = { njs_str("const") }, .type = NJS_TOKEN_CONST }, - { .entry = { njs_str("debugger") }, .type = NJS_TOKEN_DEBUGGER }, - { .entry = { njs_str("enum") }, .type = NJS_TOKEN_ENUM }, - { .entry = { njs_str("extends") }, .type = NJS_TOKEN_EXTENDS }, - { .entry = { njs_str("implements") }, .type = NJS_TOKEN_IMPLEMENTS }, - { .entry = { njs_str("interface") }, .type = NJS_TOKEN_INTERFACE }, - { .entry = { njs_str("let") }, .type = NJS_TOKEN_LET }, - { .entry = { njs_str("package") }, .type = NJS_TOKEN_PACKAGE }, - { .entry = { njs_str("private") }, .type = NJS_TOKEN_PRIVATE }, - { .entry = { njs_str("protected") }, .type = NJS_TOKEN_PROTECTED }, - { .entry = { njs_str("public") }, .type = NJS_TOKEN_PUBLIC }, - { .entry = { njs_str("static") }, .type = NJS_TOKEN_STATIC }, - { .entry = { njs_str("super") }, .type = NJS_TOKEN_SUPER }, + { + .entry = { njs_str("arguments") }, + .type = NJS_TOKEN_ARGUMENTS, + .reserved = 0 + }, + + { + .entry = { njs_str("async") }, + .type = NJS_TOKEN_ASYNC, + .reserved = 0 + }, + + { + .entry = { njs_str("await") }, + .type = NJS_TOKEN_AWAIT, + .reserved = 1 + }, + + { + .entry = { njs_str("break") }, + .type = NJS_TOKEN_BREAK, + .reserved = 1 + }, + + { + .entry = { njs_str("case") }, + .type = NJS_TOKEN_CASE, + .reserved = 1 + }, + + { + .entry = { njs_str("catch") }, + .type = NJS_TOKEN_CATCH, + .reserved = 1 + }, + + { + .entry = { njs_str("class") }, + .type = NJS_TOKEN_CLASS, + .reserved = 1 + }, + + { + .entry = { njs_str("const") }, + .type = NJS_TOKEN_CONST, + .reserved = 1 + }, + + { + .entry = { njs_str("continue") }, + .type = NJS_TOKEN_CONTINUE, + .reserved = 1 + }, + + { + .entry = { njs_str("debugger") }, + .type = NJS_TOKEN_DEBUGGER, + .reserved = 1 + }, + + { + .entry = { njs_str("default") }, + .type = NJS_TOKEN_DEFAULT, + .reserved = 1 + }, + + { + .entry = { njs_str("delete") }, + .type = NJS_TOKEN_DELETE, + .reserved = 1 + }, + + { + .entry = { njs_str("do") }, + .type = NJS_TOKEN_DO, + .reserved = 1 + }, + + { + .entry = { njs_str("else") }, + .type = NJS_TOKEN_ELSE, + .reserved = 1 + }, + + { + .entry = { njs_str("enum") }, + .type = NJS_TOKEN_ENUM, + .reserved = 1 + }, + + { + .entry = { njs_str("eval") }, + .type = NJS_TOKEN_EVAL, + .reserved = 0 + }, + + { + .entry = { njs_str("export") }, + .type = NJS_TOKEN_EXPORT, + .reserved = 1 + }, + + { + .entry = { njs_str("extends") }, + .type = NJS_TOKEN_EXTENDS, + .reserved = 1 + }, + + { + .entry = { njs_str("false") }, + .type = NJS_TOKEN_FALSE, + .reserved = 1 + }, + + { + .entry = { njs_str("finally") }, + .type = NJS_TOKEN_FINALLY, + .reserved = 1 + }, + + { + .entry = { njs_str("for") }, + .type = NJS_TOKEN_FOR, + .reserved = 1 + }, + + { + .entry = { njs_str("from") }, + .type = NJS_TOKEN_FROM, + .reserved = 0 + }, + + { + .entry = { njs_str("function") }, + .type = NJS_TOKEN_FUNCTION, + .reserved = 1 + }, + + { + .entry = { njs_str("if") }, + .type = NJS_TOKEN_IF, + .reserved = 1 + }, + + { + .entry = { njs_str("implements") }, + .type = NJS_TOKEN_IMPLEMENTS, + .reserved = 0 + }, + + { + .entry = { njs_str("import") }, + .type = NJS_TOKEN_IMPORT, + .reserved = 1 + }, + + { + .entry = { njs_str("in") }, + .type = NJS_TOKEN_IN, + .reserved = 1 + }, + + { + .entry = { njs_str("instanceof") }, + .type = NJS_TOKEN_INSTANCEOF, + .reserved = 1 + }, + + { + .entry = { njs_str("interface") }, + .type = NJS_TOKEN_INTERFACE, + .reserved = 0 + }, + + { + .entry = { njs_str("let") }, + .type = NJS_TOKEN_LET, + .reserved = 0 + }, + + { + .entry = { njs_str("meta") }, + .type = NJS_TOKEN_META, + .reserved = 0 + }, + + { + .entry = { njs_str("new") }, + .type = NJS_TOKEN_NEW, + .reserved = 1 + }, + + { + .entry = { njs_str("null") }, + .type = NJS_TOKEN_NULL, + .reserved = 1 + }, + + { + .entry = { njs_str("of") }, + .type = NJS_TOKEN_OF, + .reserved = 0 + }, + + { + .entry = { njs_str("package") }, + .type = NJS_TOKEN_PACKAGE, + .reserved = 0 + }, + + { + .entry = { njs_str("private") }, + .type = NJS_TOKEN_PRIVATE, + .reserved = 0 + }, + + { + .entry = { njs_str("protected") }, + .type = NJS_TOKEN_PROTECTED, + .reserved = 0 + }, + + { + .entry = { njs_str("public") }, + .type = NJS_TOKEN_PUBLIC, + .reserved = 0 + }, + + { + .entry = { njs_str("return") }, + .type = NJS_TOKEN_RETURN, + .reserved = 1 + }, + + { + .entry = { njs_str("static") }, + .type = NJS_TOKEN_STATIC, + .reserved = 0 + }, + + { + .entry = { njs_str("super") }, + .type = NJS_TOKEN_SUPER, + .reserved = 1 + }, + + { + .entry = { njs_str("switch") }, + .type = NJS_TOKEN_SWITCH, + .reserved = 1 + }, + + { + .entry = { njs_str("target") }, + .type = NJS_TOKEN_TARGET, + .reserved = 0 + }, + + { + .entry = { njs_str("this") }, + .type = NJS_TOKEN_THIS, + .reserved = 1 + }, + + { + .entry = { njs_str("throw") }, + .type = NJS_TOKEN_THROW, + .reserved = 1 + }, + + { + .entry = { njs_str("true") }, + .type = NJS_TOKEN_TRUE, + .reserved = 1 + }, + + { + .entry = { njs_str("try") }, + .type = NJS_TOKEN_TRY, + .reserved = 1 + }, + + { + .entry = { njs_str("typeof") }, + .type = NJS_TOKEN_TYPEOF, + .reserved = 1 + }, + + { + .entry = { njs_str("var") }, + .type = NJS_TOKEN_VAR, + .reserved = 1 + }, + + { + .entry = { njs_str("void") }, + .type = NJS_TOKEN_VOID, + .reserved = 1 + }, + + { + .entry = { njs_str("while") }, + .type = NJS_TOKEN_WHILE, + .reserved = 1 + }, + + { + .entry = { njs_str("with") }, + .type = NJS_TOKEN_WITH, + .reserved = 1 + }, + + { + .entry = { njs_str("yield") }, + .type = NJS_TOKEN_YIELD, + .reserved = 1 + }, }; -static const njs_lexer_keyword_entry_t njs_lexer_keyword_entries[75] = +static const njs_lexer_keyword_entry_t njs_lexer_keyword_entries[99] = { - { NULL, NULL, 74, 0 }, - { "case", &njs_lexer_kws[19], 4, 0 }, - { "continue", &njs_lexer_kws[17], 8, 0 }, - { "do", &njs_lexer_kws[14], 2, 0 }, - { "enum", &njs_lexer_kws[37], 4, 0 }, - { "extends", &njs_lexer_kws[38], 7, 0 }, - { "instanceof", &njs_lexer_kws[5], 10, 0 }, - { "public", &njs_lexer_kws[45], 6, 0 }, - { "static", &njs_lexer_kws[46], 6, 0 }, - { "in", &njs_lexer_kws[3], 2, 0 }, - { "await", &njs_lexer_kws[33], 5, 0 }, - { "private", &njs_lexer_kws[43], 7, 0 }, + { NULL, NULL, 98, 0 }, + { "continue", &njs_lexer_kws[8], 8, 0 }, + { "finally", &njs_lexer_kws[19], 7, 0 }, + { "return", &njs_lexer_kws[38], 6, 0 }, + { "static", &njs_lexer_kws[39], 6, 0 }, + { "async", &njs_lexer_kws[1], 5, 0 }, + { "break", &njs_lexer_kws[3], 5, 0 }, + { "interface", &njs_lexer_kws[28], 9, 0 }, + { "case", &njs_lexer_kws[4], 4, 0 }, + { "import", &njs_lexer_kws[25], 6, 0 }, + { "protected", &njs_lexer_kws[36], 9, 0 }, + { "switch", &njs_lexer_kws[41], 6, 0 }, + { "catch", &njs_lexer_kws[5], 5, 1 }, + { "delete", &njs_lexer_kws[11], 6, 0 }, + { "else", &njs_lexer_kws[13], 4, 0 }, + { "private", &njs_lexer_kws[35], 7, 0 }, + { "extends", &njs_lexer_kws[17], 7, 0 }, + { "this", &njs_lexer_kws[43], 4, 0 }, + { "false", &njs_lexer_kws[18], 5, 0 }, + { "await", &njs_lexer_kws[2], 5, 0 }, { NULL, NULL, 0, 0 }, - { "debugger", &njs_lexer_kws[36], 8, 0 }, - { "for", &njs_lexer_kws[15], 3, 1 }, + { "public", &njs_lexer_kws[37], 6, 0 }, { NULL, NULL, 0, 0 }, - { "catch", &njs_lexer_kws[25], 5, 0 }, + { "class", &njs_lexer_kws[6], 5, 0 }, + { "const", &njs_lexer_kws[7], 5, 4 }, { NULL, NULL, 0, 0 }, - { "super", &njs_lexer_kws[47], 5, 2 }, + { "try", &njs_lexer_kws[46], 3, 0 }, + { "null", &njs_lexer_kws[32], 4, 0 }, { NULL, NULL, 0, 0 }, - { "const", &njs_lexer_kws[35], 5, 0 }, + { "do", &njs_lexer_kws[12], 2, 0 }, + { "var", &njs_lexer_kws[48], 3, 0 }, + { "if", &njs_lexer_kws[23], 2, 7 }, + { "implements", &njs_lexer_kws[24], 10, 0 }, + { "with", &njs_lexer_kws[51], 4, 0 }, { NULL, NULL, 0, 0 }, - { "false", &njs_lexer_kws[1], 5, 0 }, - { "with", &njs_lexer_kws[23], 4, 0 }, - { "implements", &njs_lexer_kws[39], 10, 0 }, - { "this", &njs_lexer_kws[30], 4, 0 }, - { "let", &njs_lexer_kws[41], 3, 0 }, + { "eval", &njs_lexer_kws[15], 4, 9 }, + { NULL, NULL, 0, 0 }, + { "target", &njs_lexer_kws[42], 6, 0 }, + { "enum", &njs_lexer_kws[14], 4, 10 }, + { "instanceof", &njs_lexer_kws[27], 10, 0 }, + { NULL, NULL, 0, 0 }, + { "debugger", &njs_lexer_kws[9], 8, 0 }, { NULL, NULL, 0, 0 }, { NULL, NULL, 0, 0 }, - { "true", &njs_lexer_kws[2], 4, 0 }, + { "default", &njs_lexer_kws[10], 7, 0 }, + { "void", &njs_lexer_kws[49], 4, 0 }, + { NULL, NULL, 0, 0 }, { NULL, NULL, 0, 0 }, - { "export", &njs_lexer_kws[29], 6, 0 }, { NULL, NULL, 0, 0 }, - { "interface", &njs_lexer_kws[40], 9, 0 }, + { "from", &njs_lexer_kws[21], 4, 0 }, + { "package", &njs_lexer_kws[34], 7, 15 }, { NULL, NULL, 0, 0 }, - { "eval", &njs_lexer_kws[32], 4, 0 }, - { "protected", &njs_lexer_kws[44], 9, 0 }, - { "while", &njs_lexer_kws[13], 5, 0 }, + { "yield", &njs_lexer_kws[52], 5, 0 }, { NULL, NULL, 0, 0 }, - { "void", &njs_lexer_kws[6], 4, 0 }, + { NULL, NULL, 0, 0 }, + { "of", &njs_lexer_kws[33], 2, 0 }, { NULL, NULL, 0, 0 }, - { "return", &njs_lexer_kws[22], 6, 0 }, + { "function", &njs_lexer_kws[22], 8, 0 }, { NULL, NULL, 0, 0 }, - { "delete", &njs_lexer_kws[8], 6, 0 }, - { "yield", &njs_lexer_kws[9], 5, 0 }, - { "null", &njs_lexer_kws[0], 4, 0 }, - { "throw", &njs_lexer_kws[27], 5, 0 }, + { "true", &njs_lexer_kws[45], 4, 16 }, + { "new", &njs_lexer_kws[31], 3, 0 }, + { "export", &njs_lexer_kws[16], 6, 0 }, { NULL, NULL, 0, 0 }, { NULL, NULL, 0, 0 }, { NULL, NULL, 0, 0 }, { NULL, NULL, 0, 0 }, - { "import", &njs_lexer_kws[28], 6, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { "for", &njs_lexer_kws[20], 3, 0 }, + { "while", &njs_lexer_kws[50], 5, 0 }, { NULL, NULL, 0, 0 }, - { "switch", &njs_lexer_kws[18], 6, 0 }, - { "try", &njs_lexer_kws[24], 3, 0 }, - { "function", &njs_lexer_kws[21], 8, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, { NULL, NULL, 0, 0 }, - { "if", &njs_lexer_kws[11], 2, 0 }, - { "break", &njs_lexer_kws[16], 5, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, { NULL, NULL, 0, 0 }, { NULL, NULL, 0, 0 }, { NULL, NULL, 0, 0 }, - { "var", &njs_lexer_kws[10], 3, 4 }, + { "typeof", &njs_lexer_kws[47], 6, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { "super", &njs_lexer_kws[40], 5, 0 }, { NULL, NULL, 0, 0 }, - { "default", &njs_lexer_kws[20], 7, 0 }, - { "arguments", &njs_lexer_kws[31], 9, 6 }, - { "finally", &njs_lexer_kws[26], 7, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { "let", &njs_lexer_kws[29], 3, 19 }, + { "in", &njs_lexer_kws[26], 2, 0 }, { NULL, NULL, 0, 0 }, - { "else", &njs_lexer_kws[12], 4, 0 }, - { "class", &njs_lexer_kws[34], 5, 7 }, - { "new", &njs_lexer_kws[7], 3, 8 }, + { NULL, NULL, 0, 0 }, + { "throw", &njs_lexer_kws[44], 5, 0 }, + { "arguments", &njs_lexer_kws[0], 9, 0 }, + { "meta", &njs_lexer_kws[30], 4, 0 }, { NULL, NULL, 0, 0 }, - { "package", &njs_lexer_kws[42], 7, 11 }, - { "typeof", &njs_lexer_kws[4], 6, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, { NULL, NULL, 0, 0 }, }; diff -r 1d071c0e23e8 -r 86f55a7dc4a4 src/njs_module.c --- a/src/njs_module.c Fri May 29 14:49:36 2020 +0300 +++ b/src/njs_module.c Fri May 29 20:00:32 2020 +0300 @@ -15,6 +15,19 @@ typedef struct { } njs_module_info_t; +typedef struct {

2 years, 2 months

[njs] Added return value check forgotten in 9beb9ea093b5.

by Alexander Borisov

details: https://hg.nginx.org/njs/rev/1d071c0e23e8 branches: changeset: 1406:1d071c0e23e8 user: Alexander Borisov <alexander.borisov(a)nginx.com> date: Fri May 29 14:49:36 2020 +0300 description: Added return value check forgotten in 9beb9ea093b5. Found by Coverity (CID 1463859, 1463860). diffstat: src/njs_json.c | 12 ++++++++---- 1 files changed, 8 insertions(+), 4 deletions(-) diffs (24 lines): diff -r 9beb9ea093b5 -r 1d071c0e23e8 src/njs_json.c --- a/src/njs_json.c Wed May 27 18:18:40 2020 +0300 +++ b/src/njs_json.c Fri May 29 14:49:36 2020 +0300 @@ -1037,12 +1037,16 @@ njs_json_parse_iterator_call(njs_vm_t *v } if (njs_is_undefined(&parse->retval)) { - njs_value_property_i64_delete(vm, &state->value, state->index - 1, - NULL); + ret = njs_value_property_i64_delete(vm, &state->value, + state->index - 1, NULL); } else { - njs_value_property_i64_set(vm, &state->value, state->index - 1, - &parse->retval); + ret = njs_value_property_i64_set(vm, &state->value, + state->index - 1, &parse->retval); + } + + if (njs_slow_path(ret == NJS_ERROR)) { + return NJS_ERROR; } break;

2 years, 2 months

Requesting Help to Compile NGINX + LuaJIT STATIC Build

by BALAJI POTHULA

HI..! Team, I try to Compile NGINX + LuaJIT as STATIC Build, I want to Embedding LuaJIT inside the NGINX. The below script will generate NGINX with size 17MB. https://github.com/balajipothula/nginx/blob/master/compile-nginx-on-ubuntu1… . I tried with OpenResty source also. The below script will generate NGINX with LuaJIT (Dynamic Linking) with size 47MB. https://github.com/balajipothula/nginx/blob/master/compile-nginx-lua-on-ubu… I am expecting a Lean NGINX + LuaJIT only as a STATIC Build. Drag and Drop the nginx folder, It should work like a charm (If possible any Linux Family OS) I am trying to add More Power to NGINX with luaposix, same time NGINX should be Lean. Could you please Help me. nginx ├── bin │ └── nginx ├── client_body_temp ├── conf │ ├── mime.types │ └── nginx.conf ├── fastcgi_temp ├── html │ ├── 50x.html │ └── index.html ├── log │ ├── access.log │ ├── error.log │ └── nginx.pid ├── proxy_temp ├── scgi_temp └── uwsgi_temp Thanks and Regards, BALAJI.

2 years, 2 months

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

nginx-devel May 2020