[njs] Fixed automatic semicolon insertion.

Dmitry Volyntsev xeioex at nginx.com
Tue Nov 27 16:28:02 UTC 2018


details:   https://hg.nginx.org/njs/rev/4e62b7a295e0
branches:  
changeset: 674:4e62b7a295e0
user:      Dmitry Volyntsev <xeioex at nginx.com>
date:      Tue Nov 27 18:58:30 2018 +0300
description:
Fixed automatic semicolon insertion.

diffstat:

 njs/njs_lexer.c             |   8 +++++
 njs/njs_parser.c            |  23 +++-----------
 njs/njs_parser.h            |   6 +--
 njs/njs_parser_expression.c |  63 ++++-------------------------------------
 njs/test/njs_unit_test.c    |  68 +++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 90 insertions(+), 78 deletions(-)

diffs (351 lines):

diff -r b529a8b72e3b -r 4e62b7a295e0 njs/njs_lexer.c
--- a/njs/njs_lexer.c	Tue Nov 27 18:58:26 2018 +0300
+++ b/njs/njs_lexer.c	Tue Nov 27 18:58:30 2018 +0300
@@ -277,6 +277,7 @@ njs_lexer_token(njs_lexer_t *lexer)
 {
     njs_token_t  token;
 
+    lexer->prev_start = lexer->start;
     lexer->prev_token = lexer->token;
 
     token = njs_lexer_next_token(lexer);
@@ -287,6 +288,13 @@ njs_lexer_token(njs_lexer_t *lexer)
 }
 
 
+void
+njs_lexer_rollback(njs_lexer_t *lexer)
+{
+    lexer->start = lexer->prev_start;
+}
+
+
 static njs_token_t
 njs_lexer_next_token(njs_lexer_t *lexer)
 {
diff -r b529a8b72e3b -r 4e62b7a295e0 njs/njs_parser.c
--- a/njs/njs_parser.c	Tue Nov 27 18:58:26 2018 +0300
+++ b/njs/njs_parser.c	Tue Nov 27 18:58:30 2018 +0300
@@ -10,17 +10,6 @@
 #include <stdio.h>
 
 
-/*
- * The LL(2) parser.  The two lookahead tokens are required because
- * JavaScript inserts automatically semicolon at the end of line in
- *    a = 1
- *    b = 2
- * whilst
- *    a = 1
- *    + b
- * is treated as a single expiression.
- */
-
 static njs_ret_t njs_parser_scope_begin(njs_vm_t *vm, njs_parser_t *parser,
     njs_scope_t type);
 static void njs_parser_scope_end(njs_vm_t *vm, njs_parser_t *parser);
@@ -282,9 +271,7 @@ njs_parser_statement_chain(njs_vm_t *vm,
             node->right = parser->node;
             parser->node = node;
 
-            while (token == NJS_TOKEN_SEMICOLON
-                   || token == NJS_TOKEN_LINE_END)
-            {
+            while (token == NJS_TOKEN_SEMICOLON) {
                 token = njs_parser_token(parser);
                 if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
                     break;
@@ -1811,7 +1798,7 @@ njs_parser_terminal(njs_vm_t *vm, njs_pa
 
     if (token == NJS_TOKEN_OPEN_PARENTHESIS) {
 
-        token = njs_lexer_token(parser->lexer);
+        token = njs_parser_token(parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -2109,7 +2096,7 @@ njs_parser_terminal(njs_vm_t *vm, njs_pa
 
     parser->node = node;
 
-    return njs_lexer_token(parser->lexer);
+    return njs_parser_token(parser);
 }
 
 
@@ -2142,7 +2129,7 @@ njs_parser_builtin_object(njs_vm_t *vm, 
     parser->node = node;
     parser->code_size += sizeof(njs_vmcode_object_copy_t);
 
-    return njs_lexer_token(parser->lexer);
+    return njs_parser_token(parser);
 }
 
 
@@ -2175,7 +2162,7 @@ njs_parser_builtin_function(njs_vm_t *vm
     parser->node = node;
     parser->code_size += sizeof(njs_vmcode_object_copy_t);
 
-    return njs_lexer_token(parser->lexer);
+    return njs_parser_token(parser);
 }
 
 
diff -r b529a8b72e3b -r 4e62b7a295e0 njs/njs_parser.h
--- a/njs/njs_parser.h	Tue Nov 27 18:58:26 2018 +0300
+++ b/njs/njs_parser.h	Tue Nov 27 18:58:30 2018 +0300
@@ -31,8 +31,6 @@ typedef enum {
     NJS_TOKEN_DOT,
     NJS_TOKEN_SEMICOLON,
 
-#define NJS_TOKEN_FIRST_OPERATOR    NJS_TOKEN_COLON
-
     NJS_TOKEN_COLON,
     NJS_TOKEN_CONDITIONAL,
 
@@ -103,8 +101,6 @@ typedef enum {
     NJS_TOKEN_DELETE,
     NJS_TOKEN_YIELD,
 
-#define NJS_TOKEN_LAST_OPERATOR   NJS_TOKEN_YIELD
-
     NJS_TOKEN_DIGIT,
     NJS_TOKEN_LETTER,
 
@@ -223,6 +219,7 @@ typedef struct {
     nxt_lvlhsh_t                    keywords_hash;
 
     u_char                          *start;
+    u_char                          *prev_start;
     u_char                          *end;
 } njs_lexer_t;
 
@@ -360,6 +357,7 @@ typedef struct {
 
 
 njs_token_t njs_lexer_token(njs_lexer_t *lexer);
+void njs_lexer_rollback(njs_lexer_t *lexer);
 nxt_int_t njs_lexer_keywords_init(nxt_mem_cache_pool_t *mcp,
     nxt_lvlhsh_t *hash);
 njs_token_t njs_lexer_keyword(njs_lexer_t *lexer);
diff -r b529a8b72e3b -r 4e62b7a295e0 njs/njs_parser_expression.c
--- a/njs/njs_parser_expression.c	Tue Nov 27 18:58:26 2018 +0300
+++ b/njs/njs_parser_expression.c	Tue Nov 27 18:58:30 2018 +0300
@@ -230,14 +230,6 @@ njs_parser_expression(njs_vm_t *vm, njs_
 }
 
 
-nxt_inline nxt_bool_t
-njs_parser_expression_operator(njs_token_t token)
-{
-    return (token >= NJS_TOKEN_FIRST_OPERATOR
-            && token <= NJS_TOKEN_LAST_OPERATOR);
-}
-
-
 njs_token_t
 njs_parser_var_expression(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token)
 {
@@ -259,18 +251,6 @@ njs_parser_var_expression(njs_vm_t *vm, 
             size = sizeof(njs_vmcode_move_t);
             break;
 
-        case NJS_TOKEN_LINE_END:
-            token = njs_lexer_token(parser->lexer);
-            if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
-                return token;
-            }
-
-            if (njs_parser_expression_operator(token)) {
-                continue;
-            }
-
-            /* Fall through. */
-
         default:
             return token;
         }
@@ -397,18 +377,6 @@ njs_parser_assignment_expression(njs_vm_
             operation = njs_vmcode_bitwise_or;
             break;
 
-        case NJS_TOKEN_LINE_END:
-            token = njs_lexer_token(parser->lexer);
-            if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
-                return token;
-            }
-
-            if (njs_parser_expression_operator(token)) {
-                continue;
-            }
-
-            /* Fall through. */
-
         default:
             return token;
         }
@@ -588,18 +556,6 @@ njs_parser_binary_expression(njs_vm_t *v
 
         } while (n != 0);
 
-        if (token == NJS_TOKEN_LINE_END) {
-
-            token = njs_lexer_token(parser->lexer);
-            if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
-                return token;
-            }
-
-            if (njs_parser_expression_operator(token)) {
-                continue;
-            }
-        }
-
         return token;
 
     found:
@@ -676,18 +632,6 @@ njs_parser_exponential_expression(njs_vm
             parser->node = node;
         }
 
-        if (token == NJS_TOKEN_LINE_END) {
-
-            token = njs_lexer_token(parser->lexer);
-            if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
-                return token;
-            }
-
-            if (njs_parser_expression_operator(token)) {
-                continue;
-            }
-        }
-
         return token;
     }
 }
@@ -902,6 +846,13 @@ njs_parser_post_inc_dec_expression(njs_v
         return token;
     }
 
+    /* Automatic semicolon insertion. */
+
+    if (parser->lexer->prev_token == NJS_TOKEN_LINE_END) {
+        njs_lexer_rollback(parser->lexer);
+        return NJS_TOKEN_SEMICOLON;
+    }
+
     if (!njs_parser_is_lvalue(parser->node)) {
         njs_parser_ref_error(vm, parser,
                              "Invalid left-hand side in postfix operation");
diff -r b529a8b72e3b -r 4e62b7a295e0 njs/test/njs_unit_test.c
--- a/njs/test/njs_unit_test.c	Tue Nov 27 18:58:26 2018 +0300
+++ b/njs/test/njs_unit_test.c	Tue Nov 27 18:58:30 2018 +0300
@@ -2046,6 +2046,15 @@ static njs_unit_test_t  njs_test[] =
     { nxt_string("a = 0; a \n ++"),
       nxt_string("SyntaxError: Unexpected end of input in 2") },
 
+    { nxt_string("a = 0; a \n --"),
+      nxt_string("SyntaxError: Unexpected end of input in 2") },
+
+    { nxt_string("var a = 0; a \n + 1"),
+      nxt_string("1") },
+
+    { nxt_string("var a = 0; a \n +\n 1"),
+      nxt_string("1") },
+
     { nxt_string("var a; a = 1 ? 2 \n : 3"),
       nxt_string("2") },
 
@@ -2119,6 +2128,11 @@ static njs_unit_test_t  njs_test[] =
     { nxt_string("var a \n if (!a) a = 3; a"),
       nxt_string("3") },
 
+    /* automatic semicolon insertion. */
+
+    { nxt_string("var x = 0, y = 2; x\n--\ny; [x,y]"),
+      nxt_string("0,1") },
+
     /* if. */
 
     { nxt_string("if (0);"),
@@ -2193,6 +2207,21 @@ static njs_unit_test_t  njs_test[] =
     { nxt_string("(function(){ for (var p in [1] ){ if (1) continue; else return 0; }})()"),
       nxt_string("undefined") },
 
+    { nxt_string("(function(x){ if\n(x) return -1; else return 0; })(0)"),
+      nxt_string("0") },
+
+    { nxt_string("(function(x){ if\n(\nx) return -1; else return 0; })(0)"),
+      nxt_string("0") },
+
+    { nxt_string("(function(x){ if\n(\nx)\nreturn -1; else return 0; })(0)"),
+      nxt_string("0") },
+
+    { nxt_string("(function(x){ if\n(\nx)\nreturn -1\n else return 0; })(0)"),
+      nxt_string("0") },
+
+    { nxt_string("(function(x){ if\n(\nx)\nreturn -1\n else\nreturn 0; })(0)"),
+      nxt_string("0") },
+
     /* do while. */
 
     { nxt_string("do { break } if (false)"),
@@ -2957,6 +2986,15 @@ static njs_unit_test_t  njs_test[] =
     { nxt_string("var a = [1,2]; a.length"),
       nxt_string("2") },
 
+    { nxt_string("[\n1]"),
+      nxt_string("1") },
+
+    { nxt_string("\n[\n1\n]"),
+      nxt_string("1") },
+
+    { nxt_string("\n[\n1\n,\n2]\n[\n0]"),
+      nxt_string("1") },
+
 #if 0
     { nxt_string("Object.create([1,2]).length"),
       nxt_string("2") },
@@ -3808,6 +3846,9 @@ static njs_unit_test_t  njs_test[] =
     { nxt_string("'a'.my"),
       nxt_string("undefined") },
 
+    { nxt_string("var a = '123'\n[2].toString();a"),
+      nxt_string("3") },
+
     /* Escape strings. */
 
     { nxt_string("'\\a \\' \\\" \\\\ \\0 \\b \\f \\n \\r \\t \\v'"),
@@ -5318,6 +5359,33 @@ static njs_unit_test_t  njs_test[] =
                  "eval"),
       nxt_string("X") },
 
+    { nxt_string("var o = {f:function(x){ return x**2}}; o.f\n(2)"),
+      nxt_string("4") },
+
+    { nxt_string("var o = {f:function(x){ return x**2}}; o\n.f\n(2)"),
+      nxt_string("4") },
+
+    { nxt_string("var o = {f:function(x){ return x**2}}; o\n.\nf\n(2)"),
+      nxt_string("4") },
+
+    { nxt_string("function f(x){ return x**2}; [f(2)\n, f\n(2),\nf\n(\n2),\nf\n(\n2\n)]"),
+      nxt_string("4,4,4,4") },
+
+    { nxt_string("function f (x){ return x**2}; f\n(2)"),
+      nxt_string("4") },
+
+    { nxt_string("function f (x){ return x**2}; f\n(\n2)"),
+      nxt_string("4") },
+
+    { nxt_string("function f (x){ return x**2}; f\n(\n2\n)"),
+      nxt_string("4") },
+
+    { nxt_string("function f (x){ return x**2}; f\n(2\n)"),
+      nxt_string("4") },
+
+    { nxt_string("function f (x){ return x**2}; f(2\n)"),
+      nxt_string("4") },
+
     /* Recursive factorial. */
 
     { nxt_string("function f(a) {"


More information about the nginx-devel mailing list