[njs] Introduced njs_lexer_token_t.
Dmitry Volyntsev
xeioex at nginx.com
Thu Mar 7 17:49:09 UTC 2019
details: https://hg.nginx.org/njs/rev/8e2cb4da5e46
branches:
changeset: 819:8e2cb4da5e46
user: hongzhidao <hongzhidao at gmail.com>
date: Fri Mar 08 00:11:23 2019 +0800
description:
Introduced njs_lexer_token_t.
diffstat:
njs/njs.c | 11 +-
njs/njs_core.h | 1 +
njs/njs_lexer.c | 140 ++++++++++++++-----------
njs/njs_lexer.h | 256 ++++++++++++++++++++++++++++++++++++++++++++++++
njs/njs_lexer_keyword.c | 15 +-
njs/njs_parser.c | 108 +++++++++++--------
njs/njs_parser.h | 237 --------------------------------------------
njs/njs_regexp.c | 9 +-
8 files changed, 414 insertions(+), 363 deletions(-)
diffs (truncated from 1286 to 1000 lines):
diff -r bfd57048376e -r 8e2cb4da5e46 njs/njs.c
--- a/njs/njs.c Wed Mar 06 19:39:21 2019 +0300
+++ b/njs/njs.c Fri Mar 08 00:11:23 2019 +0800
@@ -234,13 +234,10 @@ njs_vm_compile(njs_vm_t *vm, u_char **st
prev = vm->parser;
vm->parser = parser;
- nxt_memzero(&lexer, sizeof(njs_lexer_t));
-
- lexer.start = *start;
- lexer.end = end;
- lexer.line = 1;
- lexer.file = vm->options.file;
- lexer.keywords_hash = vm->shared->keywords_hash;
+ ret = njs_lexer_init(vm, &lexer, &vm->options.file, *start, end);
+ if (nxt_slow_path(ret != NXT_OK)) {
+ return NJS_ERROR;
+ }
parser->lexer = &lexer;
diff -r bfd57048376e -r 8e2cb4da5e46 njs/njs_core.h
--- a/njs/njs_core.h Wed Mar 06 19:39:21 2019 +0300
+++ b/njs/njs_core.h Fri Mar 08 00:11:23 2019 +0800
@@ -33,6 +33,7 @@
#include <njs.h>
#include <njs_vm.h>
#include <njs_variable.h>
+#include <njs_lexer.h>
#include <njs_parser.h>
#include <njs_generator.h>
#include <njs_function.h>
diff -r bfd57048376e -r 8e2cb4da5e46 njs/njs_lexer.c
--- a/njs/njs_lexer.c Wed Mar 06 19:39:21 2019 +0300
+++ b/njs/njs_lexer.c Fri Mar 08 00:11:23 2019 +0800
@@ -18,11 +18,15 @@ struct njs_lexer_multi_s {
};
-static njs_token_t njs_lexer_next_token(njs_lexer_t *lexer);
-static njs_token_t njs_lexer_word(njs_lexer_t *lexer, u_char c);
-static njs_token_t njs_lexer_string(njs_lexer_t *lexer, u_char quote);
-static njs_token_t njs_lexer_number(njs_lexer_t *lexer, u_char c);
-static njs_token_t njs_lexer_multi(njs_lexer_t *lexer,
+static njs_token_t njs_lexer_next_token(njs_lexer_t *lexer,
+ njs_lexer_token_t *lt);
+static njs_token_t njs_lexer_word(njs_lexer_t *lexer, njs_lexer_token_t *lt,
+ u_char c);
+static njs_token_t njs_lexer_string(njs_lexer_t *lexer, njs_lexer_token_t *lt,
+ u_char quote);
+static njs_token_t njs_lexer_number(njs_lexer_t *lexer, njs_lexer_token_t *lt,
+ u_char c);
+static njs_token_t njs_lexer_multi(njs_lexer_t *lexer, njs_lexer_token_t *lt,
njs_token_t token, nxt_uint_t n, const njs_lexer_multi_t *multi);
static njs_token_t njs_lexer_division(njs_lexer_t *lexer,
njs_token_t token);
@@ -272,27 +276,49 @@ static const njs_lexer_multi_t njs_assi
};
+nxt_int_t
+njs_lexer_init(njs_vm_t *vm, njs_lexer_t *lexer, nxt_str_t *file,
+ u_char *start, u_char *end)
+{
+ nxt_memzero(lexer, sizeof(njs_lexer_t));
+
+ lexer->file = *file;
+ lexer->start = start;
+ lexer->end = end;
+ lexer->line = 1;
+ lexer->keywords_hash = vm->shared->keywords_hash;
+
+ lexer->lexer_token = nxt_mp_alloc(vm->mem_pool, sizeof(njs_lexer_token_t));
+ if (nxt_slow_path(lexer->lexer_token == NULL)) {
+ return NXT_ERROR;
+ }
+
+ return NXT_OK;
+}
+
+
njs_token_t
njs_lexer_token(njs_lexer_t *lexer)
{
- njs_token_t token;
+ njs_lexer_token_t *lt;
+
+ lt = lexer->lexer_token;
lexer->prev_start = lexer->start;
- lexer->prev_token = lexer->token;
+ lexer->prev_token = lt->token;
- token = njs_lexer_next_token(lexer);
+ lt->token = njs_lexer_next_token(lexer, lt);
- lexer->token = token;
+ if (lt->token == NJS_TOKEN_NAME) {
+ njs_lexer_keyword(lexer, lt);
- return token;
-}
-
+ if (lexer->property) {
+ lexer->property_token = lt->token;
+ return NJS_TOKEN_NAME;
+ }
+ }
-void
-njs_lexer_rollback(njs_lexer_t *lexer)
-{
- lexer->start = lexer->prev_start;
- lexer->token = lexer->prev_token;
+ return lt->token;
}
@@ -322,14 +348,14 @@ njs_lexer_peek_token(njs_lexer_t *lexer)
static njs_token_t
-njs_lexer_next_token(njs_lexer_t *lexer)
+njs_lexer_next_token(njs_lexer_t *lexer, njs_lexer_token_t *lt)
{
u_char c, *p;
nxt_uint_t n;
njs_token_t token;
const njs_lexer_multi_t *multi;
- lexer->text.start = lexer->start;
+ lt->text.start = lexer->start;
while (lexer->start < lexer->end) {
c = *lexer->start++;
@@ -339,15 +365,15 @@ njs_lexer_next_token(njs_lexer_t *lexer)
switch (token) {
case NJS_TOKEN_SPACE:
- lexer->text.start = lexer->start;
+ lt->text.start = lexer->start;
continue;
case NJS_TOKEN_LETTER:
- return njs_lexer_word(lexer, c);
+ return njs_lexer_word(lexer, lt, c);
case NJS_TOKEN_DOUBLE_QUOTE:
case NJS_TOKEN_SINGLE_QUOTE:
- return njs_lexer_string(lexer, c);
+ return njs_lexer_string(lexer, lt, c);
case NJS_TOKEN_DOT:
p = lexer->start;
@@ -356,20 +382,20 @@ njs_lexer_next_token(njs_lexer_t *lexer)
&& njs_tokens[p[0]] == NJS_TOKEN_DOT
&& njs_tokens[p[1]] == NJS_TOKEN_DOT)
{
- lexer->text.length = (p - lexer->text.start) + 2;
+ lt->text.length = (p - lt->text.start) + 2;
lexer->start += 2;
return NJS_TOKEN_ELLIPSIS;
}
if (p == lexer->end || njs_tokens[*p] != NJS_TOKEN_DIGIT) {
- lexer->text.length = p - lexer->text.start;
+ lt->text.length = p - lt->text.start;
return NJS_TOKEN_DOT;
}
/* Fall through. */
case NJS_TOKEN_DIGIT:
- return njs_lexer_number(lexer, c);
+ return njs_lexer_number(lexer, lt, c);
case NJS_TOKEN_ASSIGNMENT:
n = nxt_nitems(njs_assignment_token),
@@ -462,7 +488,7 @@ njs_lexer_next_token(njs_lexer_t *lexer)
case NJS_TOKEN_COLON:
case NJS_TOKEN_SEMICOLON:
case NJS_TOKEN_CONDITIONAL:
- lexer->text.length = lexer->start - lexer->text.start;
+ lt->text.length = lexer->start - lt->text.start;
return token;
case NJS_TOKEN_ILLEGAL:
@@ -473,20 +499,19 @@ njs_lexer_next_token(njs_lexer_t *lexer)
multi:
- return njs_lexer_multi(lexer, token, n, multi);
+ return njs_lexer_multi(lexer, lt, token, n, multi);
}
- lexer->text.length = lexer->start - lexer->text.start;
+ lt->text.length = lexer->start - lt->text.start;
return NJS_TOKEN_END;
}
static njs_token_t
-njs_lexer_word(njs_lexer_t *lexer, u_char c)
+njs_lexer_word(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char c)
{
- u_char *p;
- njs_token_t token;
+ u_char *p;
/* TODO: UTF-8 */
@@ -508,9 +533,9 @@ njs_lexer_word(njs_lexer_t *lexer, u_cha
0x00, 0x00, 0x00, 0x00, /* 0000 0000 0000 0000 0000 0000 0000 0000 */
};
- lexer->token_line = lexer->line;
- lexer->key_hash = nxt_djb_hash_add(NXT_DJB_HASH_INIT, c);
- lexer->text.start = lexer->start - 1;
+ lt->token_line = lexer->line;
+ lt->key_hash = nxt_djb_hash_add(NXT_DJB_HASH_INIT, c);
+ lt->text.start = lexer->start - 1;
for (p = lexer->start; p < lexer->end; p++) {
c = *p;
@@ -519,31 +544,24 @@ njs_lexer_word(njs_lexer_t *lexer, u_cha
break;
}
- lexer->key_hash = nxt_djb_hash_add(lexer->key_hash, c);
+ lt->key_hash = nxt_djb_hash_add(lt->key_hash, c);
}
lexer->start = p;
- lexer->text.length = p - lexer->text.start;
-
- token = njs_lexer_keyword(lexer);
+ lt->text.length = p - lt->text.start;
- if (lexer->property) {
- lexer->property_token = token;
- return NJS_TOKEN_NAME;
- }
-
- return token;
+ return NJS_TOKEN_NAME;
}
static njs_token_t
-njs_lexer_string(njs_lexer_t *lexer, u_char quote)
+njs_lexer_string(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char quote)
{
u_char *p, c;
nxt_bool_t escape;
escape = 0;
- lexer->text.start = lexer->start;
+ lt->text.start = lexer->start;
p = lexer->start;
while (p < lexer->end) {
@@ -574,7 +592,7 @@ njs_lexer_string(njs_lexer_t *lexer, u_c
if (c == quote) {
lexer->start = p;
- lexer->text.length = (p - 1) - lexer->text.start;
+ lt->text.length = (p - 1) - lt->text.start;
if (escape == 0) {
return NJS_TOKEN_STRING;
@@ -584,19 +602,19 @@ njs_lexer_string(njs_lexer_t *lexer, u_c
}
}
- lexer->text.start--;
- lexer->text.length = p - lexer->text.start;
+ lt->text.start--;
+ lt->text.length = p - lt->text.start;
return NJS_TOKEN_UNTERMINATED_STRING;
}
static njs_token_t
-njs_lexer_number(njs_lexer_t *lexer, u_char c)
+njs_lexer_number(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char c)
{
const u_char *p;
- lexer->text.start = lexer->start - 1;
+ lt->text.start = lexer->start - 1;
p = lexer->start;
@@ -611,7 +629,7 @@ njs_lexer_number(njs_lexer_t *lexer, u_c
goto illegal_token;
}
- lexer->number = njs_number_hex_parse(&p, lexer->end);
+ lt->number = njs_number_hex_parse(&p, lexer->end);
goto done;
}
@@ -625,7 +643,7 @@ njs_lexer_number(njs_lexer_t *lexer, u_c
goto illegal_token;
}
- lexer->number = njs_number_oct_parse(&p, lexer->end);
+ lt->number = njs_number_oct_parse(&p, lexer->end);
if (p < lexer->end && (*p == '8' || *p == '9')) {
goto illegal_trailer;
@@ -643,7 +661,7 @@ njs_lexer_number(njs_lexer_t *lexer, u_c
goto illegal_token;
}
- lexer->number = njs_number_bin_parse(&p, lexer->end);
+ lt->number = njs_number_bin_parse(&p, lexer->end);
if (p < lexer->end && (*p >= '2' && *p <= '9')) {
goto illegal_trailer;
@@ -660,12 +678,12 @@ njs_lexer_number(njs_lexer_t *lexer, u_c
}
p--;
- lexer->number = njs_number_dec_parse(&p, lexer->end);
+ lt->number = njs_number_dec_parse(&p, lexer->end);
done:
lexer->start = (u_char *) p;
- lexer->text.length = p - lexer->text.start;
+ lt->text.length = p - lt->text.start;
return NJS_TOKEN_NUMBER;
@@ -675,15 +693,15 @@ illegal_trailer:
illegal_token:
- lexer->text.length = p - lexer->text.start;
+ lt->text.length = p - lt->text.start;
return NJS_TOKEN_ILLEGAL;
}
static njs_token_t
-njs_lexer_multi(njs_lexer_t *lexer, njs_token_t token, nxt_uint_t n,
- const njs_lexer_multi_t *multi)
+njs_lexer_multi(njs_lexer_t *lexer, njs_lexer_token_t *lt, njs_token_t token,
+ nxt_uint_t n, const njs_lexer_multi_t *multi)
{
u_char c;
@@ -699,7 +717,7 @@ njs_lexer_multi(njs_lexer_t *lexer, njs_
break;
}
- return njs_lexer_multi(lexer, multi->token, multi->count,
+ return njs_lexer_multi(lexer, lt, multi->token, multi->count,
multi->next);
}
@@ -709,7 +727,7 @@ njs_lexer_multi(njs_lexer_t *lexer, njs_
} while (n != 0);
}
- lexer->text.length = lexer->start - lexer->text.start;
+ lt->text.length = lexer->start - lt->text.start;
return token;
}
diff -r bfd57048376e -r 8e2cb4da5e46 njs/njs_lexer.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/njs/njs_lexer.h Fri Mar 08 00:11:23 2019 +0800
@@ -0,0 +1,256 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) NGINX, Inc.
+ */
+
+#ifndef _NJS_LEXER_H_INCLUDED_
+#define _NJS_LEXER_H_INCLUDED_
+
+
+typedef enum {
+ NJS_TOKEN_AGAIN = -2,
+ NJS_TOKEN_ERROR = -1,
+ NJS_TOKEN_ILLEGAL = 0,
+
+ NJS_TOKEN_END,
+ NJS_TOKEN_SPACE,
+ NJS_TOKEN_LINE_END,
+
+ NJS_TOKEN_DOUBLE_QUOTE,
+ NJS_TOKEN_SINGLE_QUOTE,
+
+ NJS_TOKEN_OPEN_PARENTHESIS,
+ NJS_TOKEN_CLOSE_PARENTHESIS,
+ NJS_TOKEN_OPEN_BRACKET,
+ NJS_TOKEN_CLOSE_BRACKET,
+ NJS_TOKEN_OPEN_BRACE,
+ NJS_TOKEN_CLOSE_BRACE,
+
+ NJS_TOKEN_COMMA,
+ NJS_TOKEN_DOT,
+ NJS_TOKEN_ELLIPSIS,
+ NJS_TOKEN_SEMICOLON,
+
+ NJS_TOKEN_COLON,
+ NJS_TOKEN_CONDITIONAL,
+
+ NJS_TOKEN_ASSIGNMENT,
+ NJS_TOKEN_ADDITION_ASSIGNMENT,
+ NJS_TOKEN_SUBSTRACTION_ASSIGNMENT,
+ NJS_TOKEN_MULTIPLICATION_ASSIGNMENT,
+ NJS_TOKEN_EXPONENTIATION_ASSIGNMENT,
+ NJS_TOKEN_DIVISION_ASSIGNMENT,
+ NJS_TOKEN_REMAINDER_ASSIGNMENT,
+ NJS_TOKEN_LEFT_SHIFT_ASSIGNMENT,
+ NJS_TOKEN_RIGHT_SHIFT_ASSIGNMENT,
+ NJS_TOKEN_UNSIGNED_RIGHT_SHIFT_ASSIGNMENT,
+ NJS_TOKEN_BITWISE_OR_ASSIGNMENT,
+ NJS_TOKEN_BITWISE_XOR_ASSIGNMENT,
+ NJS_TOKEN_BITWISE_AND_ASSIGNMENT,
+
+#define NJS_TOKEN_LAST_ASSIGNMENT NJS_TOKEN_BITWISE_AND_ASSIGNMENT
+
+ NJS_TOKEN_EQUAL,
+ NJS_TOKEN_STRICT_EQUAL,
+ NJS_TOKEN_NOT_EQUAL,
+ NJS_TOKEN_STRICT_NOT_EQUAL,
+
+ NJS_TOKEN_ADDITION,
+ NJS_TOKEN_UNARY_PLUS,
+ NJS_TOKEN_INCREMENT,
+ NJS_TOKEN_POST_INCREMENT,
+
+ NJS_TOKEN_SUBSTRACTION,
+ NJS_TOKEN_UNARY_NEGATION,
+ NJS_TOKEN_DECREMENT,
+ NJS_TOKEN_POST_DECREMENT,
+
+ NJS_TOKEN_MULTIPLICATION,
+
+ NJS_TOKEN_EXPONENTIATION,
+
+ NJS_TOKEN_DIVISION,
+
+ NJS_TOKEN_REMAINDER,
+
+ NJS_TOKEN_LESS,
+ NJS_TOKEN_LESS_OR_EQUAL,
+ NJS_TOKEN_LEFT_SHIFT,
+
+ NJS_TOKEN_GREATER,
+ NJS_TOKEN_GREATER_OR_EQUAL,
+ NJS_TOKEN_RIGHT_SHIFT,
+ NJS_TOKEN_UNSIGNED_RIGHT_SHIFT,
+
+ NJS_TOKEN_BITWISE_OR,
+ NJS_TOKEN_LOGICAL_OR,
+
+ NJS_TOKEN_BITWISE_XOR,
+
+ NJS_TOKEN_BITWISE_AND,
+ NJS_TOKEN_LOGICAL_AND,
+
+ NJS_TOKEN_BITWISE_NOT,
+ NJS_TOKEN_LOGICAL_NOT,
+
+ NJS_TOKEN_IN,
+ NJS_TOKEN_INSTANCEOF,
+ NJS_TOKEN_TYPEOF,
+ NJS_TOKEN_VOID,
+ NJS_TOKEN_NEW,
+ NJS_TOKEN_DELETE,
+ NJS_TOKEN_YIELD,
+
+ NJS_TOKEN_DIGIT,
+ NJS_TOKEN_LETTER,
+
+#define NJS_TOKEN_FIRST_CONST NJS_TOKEN_UNDEFINED
+
+ NJS_TOKEN_UNDEFINED,
+ NJS_TOKEN_NULL,
+ NJS_TOKEN_NUMBER,
+ NJS_TOKEN_BOOLEAN,
+ NJS_TOKEN_STRING,
+
+#define NJS_TOKEN_LAST_CONST NJS_TOKEN_STRING
+
+ NJS_TOKEN_ESCAPE_STRING,
+ NJS_TOKEN_UNTERMINATED_STRING,
+ NJS_TOKEN_NAME,
+
+ NJS_TOKEN_OBJECT,
+ NJS_TOKEN_OBJECT_VALUE,
+ NJS_TOKEN_PROPERTY,
+ NJS_TOKEN_PROPERTY_DELETE,
+
+ NJS_TOKEN_ARRAY,
+
+ NJS_TOKEN_FUNCTION,
+ NJS_TOKEN_FUNCTION_EXPRESSION,
+ NJS_TOKEN_FUNCTION_CALL,
+ NJS_TOKEN_METHOD_CALL,
+ NJS_TOKEN_ARGUMENT,
+ NJS_TOKEN_RETURN,
+
+ NJS_TOKEN_REGEXP,
+
+ NJS_TOKEN_EXTERNAL,
+
+ NJS_TOKEN_STATEMENT,
+ NJS_TOKEN_BLOCK,
+ NJS_TOKEN_VAR,
+ NJS_TOKEN_IF,
+ NJS_TOKEN_ELSE,
+ NJS_TOKEN_BRANCHING,
+ NJS_TOKEN_WHILE,
+ NJS_TOKEN_DO,
+ NJS_TOKEN_FOR,
+ NJS_TOKEN_FOR_IN,
+ NJS_TOKEN_BREAK,
+ NJS_TOKEN_CONTINUE,
+ NJS_TOKEN_SWITCH,
+ NJS_TOKEN_CASE,
+ NJS_TOKEN_DEFAULT,
+ NJS_TOKEN_WITH,
+ NJS_TOKEN_TRY,
+ NJS_TOKEN_CATCH,
+ NJS_TOKEN_FINALLY,
+ NJS_TOKEN_THROW,
+
+ NJS_TOKEN_THIS,
+ NJS_TOKEN_ARGUMENTS,
+
+#define NJS_TOKEN_FIRST_OBJECT NJS_TOKEN_GLOBAL_THIS
+
+ NJS_TOKEN_GLOBAL_THIS,
+ NJS_TOKEN_NJS,
+ NJS_TOKEN_MATH,
+ NJS_TOKEN_JSON,
+
+ NJS_TOKEN_OBJECT_CONSTRUCTOR,
+ NJS_TOKEN_ARRAY_CONSTRUCTOR,
+ NJS_TOKEN_BOOLEAN_CONSTRUCTOR,
+ NJS_TOKEN_NUMBER_CONSTRUCTOR,
+ NJS_TOKEN_STRING_CONSTRUCTOR,
+ NJS_TOKEN_FUNCTION_CONSTRUCTOR,
+ NJS_TOKEN_REGEXP_CONSTRUCTOR,
+ NJS_TOKEN_DATE_CONSTRUCTOR,
+ NJS_TOKEN_ERROR_CONSTRUCTOR,
+ NJS_TOKEN_EVAL_ERROR_CONSTRUCTOR,
+ NJS_TOKEN_INTERNAL_ERROR_CONSTRUCTOR,
+ NJS_TOKEN_RANGE_ERROR_CONSTRUCTOR,
+ NJS_TOKEN_REF_ERROR_CONSTRUCTOR,
+ NJS_TOKEN_SYNTAX_ERROR_CONSTRUCTOR,
+ NJS_TOKEN_TYPE_ERROR_CONSTRUCTOR,
+ NJS_TOKEN_URI_ERROR_CONSTRUCTOR,
+ NJS_TOKEN_MEMORY_ERROR_CONSTRUCTOR,
+
+#define NJS_TOKEN_FIRST_FUNCTION NJS_TOKEN_EVAL
+
+ NJS_TOKEN_EVAL,
+ NJS_TOKEN_TO_STRING,
+ NJS_TOKEN_IS_NAN,
+ NJS_TOKEN_IS_FINITE,
+ NJS_TOKEN_PARSE_INT,
+ NJS_TOKEN_PARSE_FLOAT,
+ NJS_TOKEN_ENCODE_URI,
+ NJS_TOKEN_ENCODE_URI_COMPONENT,
+ NJS_TOKEN_DECODE_URI,
+ NJS_TOKEN_DECODE_URI_COMPONENT,
+ NJS_TOKEN_REQUIRE,
+ NJS_TOKEN_SET_TIMEOUT,
+ NJS_TOKEN_SET_IMMEDIATE,
+ NJS_TOKEN_CLEAR_TIMEOUT,
+
+ NJS_TOKEN_RESERVED,
+} njs_token_t;
+
+
+typedef struct {
+ njs_token_t token:16;
+ uint32_t token_line;
+ uint32_t key_hash;
+ nxt_str_t text;
+ double number;
+} njs_lexer_token_t;
+
+
+typedef struct {
+ njs_lexer_token_t *lexer_token;
+
+ u_char *prev_start;
+ njs_token_t prev_token:16;
+
+ uint8_t property; /* 1 bit */
+ njs_token_t property_token:16;
+
+ uint32_t line;
+ nxt_str_t file;
+
+ nxt_lvlhsh_t keywords_hash;
+
+ u_char *start;
+ u_char *end;
+} njs_lexer_t;
+
+
+typedef struct {
+ nxt_str_t name;
+ njs_token_t token;
+ double number;
+} njs_keyword_t;
+
+
+nxt_int_t njs_lexer_init(njs_vm_t *vm, njs_lexer_t *lexer, nxt_str_t *file,
+ u_char *start, u_char *end);
+njs_token_t njs_lexer_token(njs_lexer_t *lexer);
+njs_token_t njs_lexer_peek_token(njs_lexer_t *lexer);
+nxt_int_t njs_lexer_keywords_init(nxt_mp_t *mp, nxt_lvlhsh_t *hash);
+void njs_lexer_keyword(njs_lexer_t *lexer, njs_lexer_token_t *lt);
+
+#define njs_lexer_rollback(lexer) \
+ (lexer)->start = (lexer)->prev_start
+
+
+#endif /* _NJS_LEXER_H_INCLUDED_ */
diff -r bfd57048376e -r 8e2cb4da5e46 njs/njs_lexer_keyword.c
--- a/njs/njs_lexer_keyword.c Wed Mar 06 19:39:21 2019 +0300
+++ b/njs/njs_lexer_keyword.c Fri Mar 08 00:11:23 2019 +0800
@@ -173,22 +173,19 @@ njs_lexer_keywords_init(nxt_mp_t *mp, nx
}
-njs_token_t
-njs_lexer_keyword(njs_lexer_t *lexer)
+void
+njs_lexer_keyword(njs_lexer_t *lexer, njs_lexer_token_t *lt)
{
njs_keyword_t *keyword;
nxt_lvlhsh_query_t lhq;
- lhq.key_hash = lexer->key_hash;
- lhq.key = lexer->text;
+ lhq.key_hash = lt->key_hash;
+ lhq.key = lt->text;
lhq.proto = &njs_keyword_hash_proto;
if (nxt_lvlhsh_find(&lexer->keywords_hash, &lhq) == NXT_OK) {
keyword = lhq.value;
- lexer->number = keyword->number;
-
- return keyword->token;
+ lt->token = keyword->token;
+ lt->number = keyword->number;
}
-
- return NJS_TOKEN_NAME;
}
diff -r bfd57048376e -r 8e2cb4da5e46 njs/njs_parser.c
--- a/njs/njs_parser.c Wed Mar 06 19:39:21 2019 +0300
+++ b/njs/njs_parser.c Fri Mar 08 00:11:23 2019 +0800
@@ -77,6 +77,18 @@ static njs_token_t njs_parser_unexpected
#define njs_parser_chain_top_set(parser, node) \
(parser)->scope->top = node
+#define njs_parser_text(parser) \
+ &(parser)->lexer->lexer_token->text
+
+#define njs_parser_key_hash(parser) \
+ (parser)->lexer->lexer_token->key_hash
+
+#define njs_parser_number(parser) \
+ (parser)->lexer->lexer_token->number
+
+#define njs_parser_token_line(parser) \
+ (parser)->lexer->lexer_token->token_line
+
nxt_int_t
njs_parser(njs_vm_t *vm, njs_parser_t *parser, njs_parser_t *prev)
@@ -493,16 +505,17 @@ nxt_inline njs_variable_t *
njs_parser_variable_add(njs_vm_t *vm, njs_parser_t *parser,
njs_variable_type_t type)
{
- return njs_variable_add(vm, parser->scope, &parser->lexer->text,
- parser->lexer->key_hash, type);
+ return njs_variable_add(vm, parser->scope, njs_parser_text(parser),
+ njs_parser_key_hash(parser), type);
}
nxt_inline njs_ret_t
njs_parser_variable_reference(njs_vm_t *vm, njs_parser_t *parser,
njs_parser_node_t *node, njs_reference_type_t type)
{
- return njs_variable_reference(vm, parser->scope, node, &parser->lexer->text,
- parser->lexer->key_hash, type);
+ return njs_variable_reference(vm, parser->scope, node,
+ njs_parser_text(parser),
+ njs_parser_key_hash(parser), type);
}
@@ -515,8 +528,8 @@ njs_parser_labelled_statement(njs_vm_t *
njs_token_t token;
njs_variable_t *label;
- name = parser->lexer->text;
- hash = parser->lexer->key_hash;
+ name = *njs_parser_text(parser);
+ hash = njs_parser_key_hash(parser);
label = njs_label_find(vm, parser->scope, &name, hash);
if (nxt_slow_path(label != NULL)) {
@@ -604,7 +617,7 @@ njs_parser_function_declaration(njs_vm_t
return NJS_TOKEN_ERROR;
}
- node->token_line = parser->lexer->token_line;
+ node->token_line = njs_parser_token_line(parser);
token = njs_parser_token(parser);
if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
@@ -615,7 +628,7 @@ njs_parser_function_declaration(njs_vm_t
if (token == NJS_TOKEN_ARGUMENTS || token == NJS_TOKEN_EVAL) {
njs_parser_syntax_error(vm, parser, "Identifier \"%V\" "
"is forbidden in function declaration",
- &parser->lexer->text);
+ njs_parser_text(parser));
}
return NJS_TOKEN_ILLEGAL;
@@ -664,7 +677,7 @@ njs_parser_function_expression(njs_vm_t
return NJS_TOKEN_ERROR;
}
- node->token_line = parser->lexer->token_line;
+ node->token_line = njs_parser_token_line(parser);
parser->node = node;
token = njs_parser_token(parser);
@@ -768,7 +781,7 @@ njs_parser_function_lambda(njs_vm_t *vm,
arg->index = index;
index += sizeof(njs_value_t);
- ret = njs_name_copy(vm, &arg->name, &parser->lexer->text);
+ ret = njs_name_copy(vm, &arg->name, njs_parser_text(parser));
if (nxt_slow_path(ret != NXT_OK)) {
return NJS_TOKEN_ERROR;
}
@@ -946,7 +959,7 @@ njs_parser_var_statement(njs_vm_t *vm, n
if (token == NJS_TOKEN_ARGUMENTS || token == NJS_TOKEN_EVAL) {
njs_parser_syntax_error(vm, parser, "Identifier \"%V\" "
"is forbidden in var declaration",
- &parser->lexer->text);
+ njs_parser_text(parser));
}
return NJS_TOKEN_ILLEGAL;
@@ -1292,7 +1305,7 @@ njs_parser_for_statement(njs_vm_t *vm, n
} else {
- name = parser->lexer->text;
+ name = *njs_parser_text(parser);
token = njs_parser_expression(vm, parser, token);
if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
@@ -1398,7 +1411,7 @@ njs_parser_for_var_statement(njs_vm_t *v
if (token == NJS_TOKEN_ARGUMENTS || token == NJS_TOKEN_EVAL) {
njs_parser_syntax_error(vm, parser, "Identifier \"%V\" "
"is forbidden in for-in var declaration",
- &parser->lexer->text);
+ njs_parser_text(parser));
}
return NJS_TOKEN_ILLEGAL;
@@ -1573,7 +1586,7 @@ njs_parser_brk_statement(njs_vm_t *vm, n
return NJS_TOKEN_ERROR;
}
- node->token_line = parser->lexer->token_line;
+ node->token_line = njs_parser_token_line(parser);
parser->node = node;
token = njs_lexer_token(parser->lexer);
@@ -1584,8 +1597,8 @@ njs_parser_brk_statement(njs_vm_t *vm, n
return njs_parser_token(parser);
case NJS_TOKEN_NAME:
- name = parser->lexer->text;
- hash = parser->lexer->key_hash;
+ name = *njs_parser_text(parser);
+ hash = njs_parser_key_hash(parser);
if (njs_label_find(vm, parser->scope, &name, hash) == NULL) {
njs_parser_syntax_error(vm, parser, "Undefined label \"%V\"",
@@ -1593,7 +1606,7 @@ njs_parser_brk_statement(njs_vm_t *vm, n
return NJS_TOKEN_ILLEGAL;
}
- ret = njs_name_copy(vm, &parser->node->label, &parser->lexer->text);
+ ret = njs_name_copy(vm, &parser->node->label, &name);
if (nxt_slow_path(ret != NXT_OK)) {
return NJS_TOKEN_ERROR;
}
@@ -1875,11 +1888,8 @@ njs_parser_terminal(njs_vm_t *vm, njs_pa
{
double num;
njs_ret_t ret;
- njs_lexer_t *lexer;
njs_parser_node_t *node;
- lexer = parser->lexer;
-
if (token == NJS_TOKEN_OPEN_PARENTHESIS) {
token = njs_parser_token(parser);
@@ -1952,12 +1962,12 @@ njs_parser_terminal(njs_vm_t *vm, njs_pa
return token;
}
- nxt_thread_log_debug("REGEX: '%V'", &lexer->text);
+ nxt_thread_log_debug("REGEX: '%V'", njs_parser_text(parser));
break;
case NJS_TOKEN_STRING:
- nxt_thread_log_debug("JS: '%V'", &lexer->text);
+ nxt_thread_log_debug("JS: '%V'", njs_parser_text(parser));
node = njs_parser_node_new(vm, parser, NJS_TOKEN_STRING);
if (nxt_slow_path(node == NULL)) {
@@ -1972,7 +1982,7 @@ njs_parser_terminal(njs_vm_t *vm, njs_pa
break;
case NJS_TOKEN_ESCAPE_STRING:
- nxt_thread_log_debug("JS: '%V'", &lexer->text);
+ nxt_thread_log_debug("JS: '%V'", njs_parser_text(parser));
node = njs_parser_node_new(vm, parser, NJS_TOKEN_STRING);
if (nxt_slow_path(node == NULL)) {
@@ -1988,12 +1998,12 @@ njs_parser_terminal(njs_vm_t *vm, njs_pa
case NJS_TOKEN_UNTERMINATED_STRING:
njs_parser_syntax_error(vm, parser, "Unterminated string \"%V\"",
- &lexer->text);
+ njs_parser_text(parser));
return NJS_TOKEN_ILLEGAL;
case NJS_TOKEN_NUMBER:
- num = lexer->number;
+ num = njs_parser_number(parser);
nxt_thread_log_debug("JS: %f", num);
node = njs_parser_node_new(vm, parser, NJS_TOKEN_NUMBER);
@@ -2008,14 +2018,15 @@ njs_parser_terminal(njs_vm_t *vm, njs_pa
break;
case NJS_TOKEN_BOOLEAN:
- nxt_thread_log_debug("JS: boolean: %V", &lexer->text);
+ num = njs_parser_number(parser);
+ nxt_thread_log_debug("JS: boolean: %V", njs_parser_text(parser));
node = njs_parser_node_new(vm, parser, NJS_TOKEN_BOOLEAN);
if (nxt_slow_path(node == NULL)) {
return NJS_TOKEN_ERROR;
}
- if (lexer->number == 0) {
+ if (num == 0) {
node->u.value = njs_value_false;
} else {
@@ -2025,8 +2036,9 @@ njs_parser_terminal(njs_vm_t *vm, njs_pa
break;
default:
- node = njs_parser_reference(vm, parser, token, &lexer->text,
- lexer->key_hash);
+ node = njs_parser_reference(vm, parser, token,
+ njs_parser_text(parser),
+ njs_parser_key_hash(parser));
if (nxt_slow_path(node == NULL)) {
return NJS_TOKEN_ERROR;
@@ -2205,7 +2217,7 @@ njs_parser_reference(njs_vm_t *vm, njs_p
case NJS_TOKEN_NAME:
nxt_thread_log_debug("JS: %V", name);
- node->token_line = parser->lexer->token_line;
+ node->token_line = njs_parser_token_line(parser);
ext = njs_external_lookup(vm, name, hash);
@@ -2319,8 +2331,8 @@ njs_parser_object(njs_vm_t *vm, njs_pars
return njs_parser_token(parser);
case NJS_TOKEN_NAME:
- name = lexer->text;
- hash = lexer->key_hash;
+ name = *njs_parser_text(parser);
+ hash = njs_parser_key_hash(parser);
token = njs_parser_token(parser);
break;
@@ -2504,7 +2516,7 @@ njs_parser_string_create(njs_vm_t *vm, n
ssize_t length;
nxt_str_t *src;
- src = &vm->parser->lexer->text;
+ src = njs_parser_text(vm->parser);
length = nxt_utf8_length(src->start, src->length);
@@ -2535,6 +2547,7 @@ njs_parser_escape_string_create(njs_vm_t
u_char c, *start, *dst;
size_t size,length, hex_length;
uint64_t u;
+ nxt_str_t *string;
const u_char *p, *src, *end, *hex_end;
start = NULL;
@@ -2549,8 +2562,9 @@ njs_parser_escape_string_create(njs_vm_t
size = 0;
length = 0;
- src = parser->lexer->text.start;
- end = src + parser->lexer->text.length;
+ string = njs_parser_text(parser);
+ src = string->start;
+ end = src + string->length;
while (src < end) {
c = *src++;
@@ -2700,7 +2714,7 @@ njs_parser_escape_string_create(njs_vm_t
invalid:
njs_parser_syntax_error(vm, parser, "Invalid Unicode code point \"%V\"",
- &parser->lexer->text);
+ njs_parser_text(parser));
return NJS_TOKEN_ILLEGAL;
}
@@ -2743,7 +2757,7 @@ njs_parser_unexpected_token(njs_vm_t *vm
{
if (token != NJS_TOKEN_END) {
njs_parser_syntax_error(vm, parser, "Unexpected token \"%V\"",
- &parser->lexer->text);
+ njs_parser_text(parser));
} else {
njs_parser_syntax_error(vm, parser, "Unexpected end of input");
@@ -2757,10 +2771,11 @@ u_char *
njs_parser_trace_handler(nxt_trace_t *trace, nxt_trace_data_t *td,
u_char *start)
{
- u_char *p;
- size_t size;
- njs_vm_t *vm;
- njs_lexer_t *lexer;
+ u_char *p;
+ size_t size;
+ njs_vm_t *vm;
+ njs_lexer_t *lexer;
+ njs_parser_t *parser;
size = nxt_length("InternalError: ");
memcpy(start, "InternalError: ", size);
@@ -2771,14 +2786,17 @@ njs_parser_trace_handler(nxt_trace_t *tr
trace = trace->next;
p = trace->handler(trace, td, p);
- if (vm->parser != NULL && vm->parser->lexer != NULL) {
- lexer = vm->parser->lexer;
+ parser = vm->parser;
+
+ if (parser != NULL && parser->lexer != NULL) {
+ lexer = parser->lexer;
if (lexer->file.length != 0) {
njs_internal_error(vm, "%s in %V:%uD", start, &lexer->file,
- lexer->line);
+ njs_parser_token_line(parser));
} else {
- njs_internal_error(vm, "%s in %uD", start, lexer->line);
+ njs_internal_error(vm, "%s in %uD", start,
+ njs_parser_token_line(parser));
}
} else {
diff -r bfd57048376e -r 8e2cb4da5e46 njs/njs_parser.h
More information about the nginx-devel
mailing list