Parser: own the [In] grammar parameter

Previously, the [In] parameter lived in the lexer, in a byte array indexed by
bracket nesting depth.  The lexer advanced the depth while prereading, the
parser wrote at the depth it had consumed to, so a delimiter in a conditional
consequent left the write in the wrong slot and the restriction was lost:

    for (a ? (b) : c in d; false;);

The fix is to carry the parameter in njs_parser_stack_entry_t: a frame
records the value in effect when it was pushed and restores it when popped,
including on the njs_parser_reject() landing frame.  An arrow initializer now
gets [~In] as well, which it escaped before.

This also allows to remove the [In] support from the lexer. The parser
owns the parameter now. njs_lexer_t drops from 104 to 80 bytes and no longer
allocates.

diff --git a/src/njs_lexer.c b/src/njs_lexer.c
index b5c2ce121049b93666f0fcfcda84670bf96e6e93..b0448a516468407622e401b56ffe19821d9b5705 100644 (file)
--- a/src/njs_lexer.c
+++ b/src/njs_lexer.c
@@ -304,105 +304,6 @@ njs_lexer_init(njs_vm_t *vm, njs_lexer_t *lexer, njs_str_t *file,
 
     njs_queue_init(&lexer->preread);
 
-    return njs_lexer_in_stack_init(lexer);
-}
-
-
-njs_int_t
-njs_lexer_in_stack_init(njs_lexer_t *lexer)
-{
-    lexer->in_stack_size = 128;
-    lexer->in_stack = njs_mp_zalloc(lexer->vm->mem_pool, lexer->in_stack_size);
-    if (lexer->in_stack == NULL) {
-        return NJS_ERROR;
-    }
-
-    lexer->in_stack_ptr = 0;
-
-    return NJS_OK;
-}
-
-
-njs_int_t
-njs_lexer_in_stack_push(njs_lexer_t *lexer)
-{
-    u_char  *tmp;
-    size_t  size;
-
-    lexer->in_stack_ptr++;
-
-    if (lexer->in_stack_ptr < lexer->in_stack_size) {
-        lexer->in_stack[lexer->in_stack_ptr] = 0;
-        return NJS_OK;
-    }
-
-    /* Realloc in_stack, it is up to higher layer generate error if any. */
-
-    size = lexer->in_stack_size;
-    lexer->in_stack_size = size * 2;
-
-    tmp = njs_mp_alloc(lexer->vm->mem_pool, size * 2);
-    if (tmp == NULL) {
-        return NJS_ERROR;
-    }
-
-    memcpy(tmp, lexer->in_stack, size);
-    memset(&tmp[size], 0, size);
-
-    njs_mp_free(lexer->vm->mem_pool, lexer->in_stack);
-    lexer->in_stack = tmp;
-
-    return NJS_OK;
-}
-
-
-void
-njs_lexer_in_stack_pop(njs_lexer_t *lexer)
-{
-    /**
-     * if in_stack_ptr <= 0 do nothing, it is up to higher layer
-     * generate error.
-     */
-
-    if (lexer->in_stack_ptr > 0) {
-        lexer->in_stack_ptr--;
-    }
-}
-
-
-njs_int_t
-njs_lexer_in_fail_get(njs_lexer_t *lexer)
-{
-    return lexer->in_stack[lexer->in_stack_ptr];
-}
-
-
-void
-njs_lexer_in_fail_set(njs_lexer_t *lexer, njs_int_t flag)
-{
-    lexer->in_stack[lexer->in_stack_ptr] = flag;
-}
-
-
-njs_inline njs_int_t
-njs_lexer_in_stack(njs_lexer_t *lexer, njs_lexer_token_t  *token)
-{
-    switch (token->type) {
-    case NJS_TOKEN_OPEN_PARENTHESIS:
-    case NJS_TOKEN_OPEN_BRACKET:
-    case NJS_TOKEN_OPEN_BRACE:
-        return njs_lexer_in_stack_push(lexer);
-
-    case NJS_TOKEN_CLOSE_PARENTHESIS:
-    case NJS_TOKEN_CLOSE_BRACKET:
-    case NJS_TOKEN_CLOSE_BRACE:
-        njs_lexer_in_stack_pop(lexer);
-        break;
-
-    default:
-        break;
-    }
-
     return NJS_OK;
 }
 
@@ -428,11 +329,6 @@ njs_lexer_next_token(njs_lexer_t *lexer)
 
     njs_queue_insert_tail(&lexer->preread, &token->link);
 
-    ret = njs_lexer_in_stack(lexer, token);
-    if (njs_slow_path(ret != NJS_OK)) {
-        return NULL;
-    }
-
     return token;
 }
 

diff --git a/src/njs_lexer.h b/src/njs_lexer.h
index 98b8dd87d70e64b9676a83681174f3a3a261b5ab..a0b37b4bae678fa3bb63b998bd280ead683d2a30 100644 (file)
--- a/src/njs_lexer.h
+++ b/src/njs_lexer.h
@@ -266,11 +266,6 @@ typedef struct {
 
     u_char                          *start;
     u_char                          *end;
-
-#define NJS_INITIAL_IN_STACK_SIZE 128
-    uint8_t                        *in_stack;
-    njs_int_t                       in_stack_ptr;
-    njs_int_t                       in_stack_size;
 } njs_lexer_t;
 
 
@@ -283,11 +278,6 @@ njs_lexer_token_t *njs_lexer_peek_token(njs_lexer_t *lexer,
     njs_lexer_token_t *current, njs_bool_t with_end_line);
 void njs_lexer_consume_token(njs_lexer_t *lexer, unsigned length);
 njs_int_t njs_lexer_make_token(njs_lexer_t *lexer, njs_lexer_token_t *token);
-njs_int_t njs_lexer_in_stack_init(njs_lexer_t *lexer);
-njs_int_t njs_lexer_in_stack_push(njs_lexer_t *lexer);
-void njs_lexer_in_stack_pop(njs_lexer_t *lexer);
-void njs_lexer_in_fail_set(njs_lexer_t *lexer, njs_int_t flag);
-njs_int_t njs_lexer_in_fail_get(njs_lexer_t *lexer);
 
 
 const njs_lexer_keyword_entry_t *njs_lexer_keyword(const u_char *key,

diff --git a/src/njs_parser.c b/src/njs_parser.c
index debace62c69316fe2dc3b99ba61b20b19ae9bdb9..de447acd5af6e3b451b2c38d6eada32a03dd2104 100644 (file)
--- a/src/njs_parser.c
+++ b/src/njs_parser.c
@@ -547,6 +547,7 @@ njs_parser_reject(njs_parser_t *parser)
         if (!entry->optional) {
             njs_parser_next(parser, entry->state);
             parser->target = entry->node;
+            parser->allow_in = entry->allow_in;
 
             return NJS_DECLINED;
         }
@@ -602,6 +603,7 @@ njs_parser(njs_vm_t *vm, njs_parser_t *parser)
     njs_queue_init(&parser->stack);
 
     parser->target = NULL;
+    parser->allow_in = 1;
     njs_parser_next(parser, njs_parser_statement_list);
 
     ret = njs_parser_after(parser, njs_queue_first(&parser->stack),
@@ -1199,8 +1201,8 @@ njs_parser_primary_expression_test(njs_parser_t *parser,
 
         njs_parser_next(parser, njs_parser_expression);
 
-        return njs_parser_after(parser, current, NULL, 0,
-                                njs_parser_close_parenthesis);
+        return njs_parser_after_in(parser, current, NULL, 0,
+                                   njs_parser_close_parenthesis);
 
     default:
         if (njs_lexer_token_is_identifier_reference(token)) {
@@ -1446,8 +1448,8 @@ njs_parser_template_literal_string(njs_parser_t *parser,
 
     njs_lexer_consume_token(parser->lexer, 1);
 
-    return njs_parser_after(parser, current, parser->target, 0,
-                            njs_parser_template_literal_expression);
+    return njs_parser_after_in(parser, current, parser->target, 0,
+                               njs_parser_template_literal_expression);
 }
 
 
@@ -1652,6 +1654,7 @@ njs_parser_array_literal(njs_parser_t *parser, njs_lexer_token_t *token,
 {
     parser->target = parser->node;
     parser->node = NULL;
+    parser->allow_in = 1;
 
     njs_parser_next(parser, njs_parser_array_element_list);
 
@@ -1776,8 +1779,8 @@ njs_parser_object_literal(njs_parser_t *parser, njs_lexer_token_t *token,
 
     njs_parser_next(parser, njs_parser_property_definition_list);
 
-    return njs_parser_after(parser, current, node, 1,
-                            njs_parser_object_literal_after);
+    return njs_parser_after_in(parser, current, node, 1,
+                               njs_parser_object_literal_after);
 }
 
 
@@ -2306,8 +2309,8 @@ njs_parser_property(njs_parser_t *parser, njs_lexer_token_t *token,
 
         njs_parser_next(parser, njs_parser_expression);
 
-        return njs_parser_after(parser, current, node, 1,
-                                njs_parser_member_expression_bracket);
+        return njs_parser_after_in(parser, current, node, 1,
+                                   njs_parser_member_expression_bracket);
 
     case NJS_TOKEN_DOT:
         token = njs_lexer_peek_token(parser->lexer, token, 0);
@@ -3074,8 +3077,8 @@ njs_parser_arguments(njs_parser_t *parser, njs_lexer_token_t *token,
 
     njs_parser_next(parser, njs_parser_argument_list);
 
-    return njs_parser_after(parser, current, NULL, 1,
-                            njs_parser_parenthesis_or_comma);
+    return njs_parser_after_in(parser, current, NULL, 1,
+                               njs_parser_parenthesis_or_comma);
 }
 
 
@@ -4205,7 +4208,7 @@ njs_parser_relational_expression_match(njs_parser_t *parser,
         break;
 
     case NJS_TOKEN_IN:
-        if (njs_lexer_in_fail_get(parser->lexer)) {
+        if (!parser->allow_in) {
             njs_parser_syntax_error(parser, "Invalid left-hand side in for-loop");
             return NJS_ERROR;
         }
@@ -4526,14 +4529,10 @@ njs_parser_conditional_question_mark(njs_parser_t *parser,
 
     njs_lexer_consume_token(parser->lexer, 1);
 
-    if (njs_lexer_in_stack_push(parser->lexer) != NJS_OK) {
-        return NJS_ERROR;
-    }
-
     njs_parser_next(parser, njs_parser_assignment_expression);
 
-    return njs_parser_after(parser, current, cond, 1,
-                            njs_parser_conditional_colon);
+    return njs_parser_after_in(parser, current, cond, 1,
+                               njs_parser_conditional_colon);
 }
 
 
@@ -4547,8 +4546,6 @@ njs_parser_conditional_colon(njs_parser_t *parser, njs_lexer_token_t *token,
         return njs_parser_failed(parser);
     }
 
-    njs_lexer_in_stack_pop(parser->lexer);
-
     njs_lexer_consume_token(parser->lexer, 1);
 
     node = parser->target->right;
@@ -5807,7 +5804,7 @@ njs_parser_for_expression_map_continue(njs_parser_t *parser,
     njs_parser_node_t  *node;
 
     if (token->type != NJS_TOKEN_IN) {
-        njs_lexer_in_fail_set(parser->lexer, 1);
+        parser->allow_in = 0;
 
         /* Continue parsing of expr1 in "for (expr1;[expr2];[expr3])". */
 
@@ -6027,8 +6024,11 @@ njs_parser_iteration_statement_for_map(njs_parser_t *parser,
     default:
         ret = njs_parser_match_arrow_expression(parser, token);
         if (ret == NJS_OK) {
+            parser->allow_in = 0;
+
             parser->target = NULL;
             njs_parser_next(parser, njs_parser_expression);
+
             goto expression_after;
         } else if (ret == NJS_ERROR) {
             return NJS_ERROR;
@@ -6047,7 +6047,7 @@ njs_parser_iteration_statement_for_map(njs_parser_t *parser,
         case NJS_TOKEN_LOGICAL_NOT:
         case NJS_TOKEN_INCREMENT:
         case NJS_TOKEN_DECREMENT:
-            njs_lexer_in_fail_set(parser->lexer, 1);
+            parser->allow_in = 0;
 
             parser->target = NULL;
             njs_parser_next(parser, njs_parser_expression);
@@ -6146,7 +6146,7 @@ njs_parser_for_var_binding_or_var_list(njs_parser_t *parser,
             if (next->type != NJS_TOKEN_IN) {
                 parser->var_type = type;
 
-                njs_lexer_in_fail_set(parser->lexer, 1);
+                parser->allow_in = 0;
 
                 njs_parser_next(parser, njs_parser_variable_declaration_list);
                 return NJS_OK;
@@ -6291,7 +6291,7 @@ njs_parser_for_var_in_of_expression(njs_parser_t *parser,
 
     switch (token->type) {
     case NJS_TOKEN_SEMICOLON:
-        njs_lexer_in_fail_set(parser->lexer, 0);
+        parser->allow_in = 1;
 
         token = njs_lexer_peek_token(parser->lexer, token, 0);
         if (token == NULL) {
@@ -7850,8 +7850,8 @@ njs_parser_arrow_function_arrow(njs_parser_t *parser,
 
         njs_parser_next(parser, njs_parser_statement_list);
 
-        return njs_parser_after(parser, current, parser->target, 1,
-                                njs_parser_function_lambda_body_after);
+        return njs_parser_after_in(parser, current, parser->target, 1,
+                                   njs_parser_function_lambda_body_after);
     }
 
     parser->node = NULL;
@@ -8212,8 +8212,8 @@ njs_parser_function_lambda_args_after(njs_parser_t *parser,
 
     njs_parser_next(parser, njs_parser_statement_list);
 
-    return njs_parser_after(parser, current, parser->target, 1,
-                            njs_parser_function_lambda_body_after);
+    return njs_parser_after_in(parser, current, parser->target, 1,
+                               njs_parser_function_lambda_body_after);
 }
 
 
@@ -8949,11 +8949,6 @@ njs_parser_template_string(njs_parser_t *parser, njs_lexer_token_t *token)
                 p++;
                 text->length = p - text->start - 2;
 
-                ret = njs_lexer_in_stack_push(lexer);
-                if (njs_slow_path(ret != NJS_OK)) {
-                    return NJS_ERROR;
-                }
-
                 goto done;
             }
 

diff --git a/src/njs_parser.h b/src/njs_parser.h
index 1e760454dedc6598ac7cdd5404e1d4bfc11010a5..d21fcf6aae8f4dcb37eb6aac7e2dd457b178f1c2 100644 (file)
--- a/src/njs_parser.h
+++ b/src/njs_parser.h
@@ -79,6 +79,9 @@ struct njs_parser_s {
     njs_variable_type_t             var_type;
     njs_int_t                       ret;
 
+    /* The ECMAScript [In] grammar parameter. */
+    uint8_t                         allow_in;
+
     uint8_t                         use_lhs;
 
     uint8_t                         module;
@@ -94,7 +97,8 @@ typedef struct {
 
     njs_parser_node_t               *node;
 
-    njs_bool_t                      optional;
+    uint8_t                         allow_in;
+    uint8_t                         optional;
 } njs_parser_stack_entry_t;
 
 
@@ -301,6 +305,7 @@ njs_parser_stack_pop(njs_parser_t *parser)
     njs_parser_next(parser, entry->state);
 
     parser->target = entry->node;
+    parser->allow_in = entry->allow_in;
 
     njs_mp_free(parser->vm->mem_pool, entry);
 
@@ -326,6 +331,11 @@ njs_parser_stack_pop(njs_parser_t *parser)
 
 #endif
 
+
+#define njs_parser_after_in(_p, _l, _n, _opt, _state)                         \
+    njs_parser_allow_in(_p, njs_parser_after(_p, _l, _n, _opt, _state))
+
+
 njs_inline njs_int_t
 _njs_parser_after(njs_parser_t *parser, njs_queue_link_t *link, void *node,
     njs_bool_t is_optional, njs_parser_state_func_t state)
@@ -340,6 +350,7 @@ _njs_parser_after(njs_parser_t *parser, njs_queue_link_t *link, void *node,
 
     entry->state = state;
     entry->node = node;
+    entry->allow_in = parser->allow_in;
     entry->optional = is_optional;
 
     njs_queue_insert_before(link, &entry->link);
@@ -348,6 +359,21 @@ _njs_parser_after(njs_parser_t *parser, njs_queue_link_t *link, void *node,
 }
 
 
+/* Opens a [+In] region; the frame must already carry the enclosing value. */
+
+njs_inline njs_int_t
+njs_parser_allow_in(njs_parser_t *parser, njs_int_t ret)
+{
+    if (njs_slow_path(ret != NJS_OK)) {
+        return ret;
+    }
+
+    parser->allow_in = 1;
+
+    return NJS_OK;
+}
+
+
 njs_inline njs_int_t
 njs_parser_failed(njs_parser_t *parser)
 {

diff --git a/src/test/njs_unit_test.c b/src/test/njs_unit_test.c
index 639bb7a8c1a3c0b19e15077e087afc5f8c20bd6f..924dd9437ab9635b35a3b0b28517b8ba888a2feb 100644 (file)
--- a/src/test/njs_unit_test.c
+++ b/src/test/njs_unit_test.c
@@ -3494,6 +3494,50 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("for (var x = true ? 1 in {} : 0;;) break;"),
       njs_str("undefined") },
 
+    /*
+     * A delimiter in the conditional consequent used to desynchronize the
+     * [In] parameter from the parser position.
+     */
+
+    { njs_str("for (a ? (b) : c in d; false;);"),
+      njs_str("SyntaxError: Invalid left-hand side in for-loop") },
+
+    { njs_str("for (a ? b : c in d; false;);"),
+      njs_str("SyntaxError: Invalid left-hand side in for-loop") },
+
+    { njs_str("var a=1,b=2,c=3; for (a ? (b) : c; false;) break; a"),
+      njs_str("1") },
+
+    /* [+In] regions, past the speculative left-hand side parse. */
+
+    { njs_str("var a=1; for (a, (1 in {});;) break; a"),
+      njs_str("1") },
+
+    { njs_str("var a=1; for (a, `${1 in {}}`;;) break; a"),
+      njs_str("1") },
+
+    { njs_str("var a=1; for (a, function(){ return 1 in {} };;) break; a"),
+      njs_str("1") },
+
+    /* ... and [~In] must be restored after each of them. */
+
+    { njs_str("for (a, [1], b in c;;) break;"),
+      njs_str("SyntaxError: Invalid left-hand side in for-loop") },
+
+    { njs_str("for (a, `${1}`, b in c;;) break;"),
+      njs_str("SyntaxError: Invalid left-hand side in for-loop") },
+
+    /* An arrow concise body inherits [In], an arrow block body does not. */
+
+    { njs_str("for (x => 1 in {};;) break;"),
+      njs_str("SyntaxError: Invalid left-hand side in for-loop") },
+
+    { njs_str("for (x => x, 1 in {};;) break;"),
+      njs_str("SyntaxError: Invalid left-hand side in for-loop") },
+
+    { njs_str("var a=1; for (x => { return 1 in {} };;) break; a"),
+      njs_str("1") },
+
     { njs_str("for (true ? 0 in {}: 0; false; ) ;"),
       njs_str("undefined") },