From: Alexander Borisov Date: Wed, 26 Feb 2020 13:22:10 +0000 (+0300) Subject: Lexer refactoring. X-Git-Url: http://git.kaiwu.me/postgresql/log/contrib/postgres_fdw/static/gitweb.js?a=commitdiff_plain;h=7fa0c3952f2d2c89a8fa018411ad5c740e62cb83;p=njs.git Lexer refactoring. --- diff --git a/src/njs_builtin.c b/src/njs_builtin.c index 56272e2e..d948c02c 100644 --- a/src/njs_builtin.c +++ b/src/njs_builtin.c @@ -134,12 +134,6 @@ njs_builtin_objects_create(njs_vm_t *vm) } njs_lvlhsh_init(&shared->keywords_hash); - - ret = njs_lexer_keywords_init(vm->mem_pool, &shared->keywords_hash); - if (njs_slow_path(ret != NJS_OK)) { - return NJS_ERROR; - } - njs_lvlhsh_init(&shared->values_hash); pattern = njs_regexp_pattern_create(vm, (u_char *) "(?:)", @@ -495,7 +489,6 @@ njs_builtin_completions(njs_vm_t *vm) njs_arr_t *array; njs_str_t *completion; njs_int_t ret; - njs_keyword_t *keyword; njs_lvlhsh_each_t lhe; njs_builtin_traverse_t ctx; const njs_object_prop_t *prop; @@ -505,23 +498,9 @@ njs_builtin_completions(njs_vm_t *vm) return NULL; } - /* Keywords completions. */ - - njs_lvlhsh_each_init(&lhe, &njs_keyword_hash_proto); - - for ( ;; ) { - keyword = njs_lvlhsh_each(&vm->shared->keywords_hash, &lhe); - - if (keyword == NULL) { - break; - } - - completion = njs_arr_add(array); - if (njs_slow_path(completion == NULL)) { - return NULL; - } - - *completion = keyword->name; + ret = njs_lexer_keywords(array); + if (njs_slow_path(ret != NJS_OK)) { + return NULL; } /* Global object completions. */ @@ -570,12 +549,14 @@ njs_vm_completions(njs_vm_t *vm, njs_str_t *expression) static njs_arr_t * njs_vm_expression_completions(njs_vm_t *vm, njs_str_t *expression) { - u_char *p, *end; - njs_int_t ret; - njs_value_t *value; - njs_variable_t *var; - njs_object_prop_t *prop; - njs_lvlhsh_query_t lhq; + u_char *p, *end; + njs_int_t ret; + njs_value_t *value; + njs_variable_t *var; + njs_rbtree_node_t *node; + njs_object_prop_t *prop; + njs_lvlhsh_query_t lhq; + njs_variable_node_t var_node; if (njs_slow_path(vm->parser == NULL)) { return NULL; @@ -588,16 +569,23 @@ njs_vm_expression_completions(njs_vm_t *vm, njs_str_t *expression) while (p < end && *p != '.') { p++; } - lhq.proto = &njs_variables_hash_proto; + lhq.proto = &njs_lexer_hash_proto; lhq.key.length = p - lhq.key.start; lhq.key_hash = njs_djb_hash(lhq.key.start, lhq.key.length); - ret = njs_lvlhsh_find(&vm->parser->scope->variables, &lhq); + ret = njs_lvlhsh_find(&vm->shared->keywords_hash, &lhq); if (njs_slow_path(ret != NJS_OK)) { return NULL; } - var = lhq.value; + var_node.key = (uintptr_t) lhq.value; + + node = njs_rbtree_find(&vm->parser->scope->variables, &var_node.node); + if (njs_slow_path(node == NULL)) { + return NULL; + } + + var = ((njs_variable_node_t *) node)->variable; value = njs_vmcode_operand(vm, var->index); if (!njs_is_object(value)) { diff --git a/src/njs_function.c b/src/njs_function.c index 850c1f8d..bb612a52 100644 --- a/src/njs_function.c +++ b/src/njs_function.c @@ -932,7 +932,7 @@ njs_function_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, scope = parser->scope; - ret = njs_variables_copy(vm, &scope->variables, &vm->variables_hash); + ret = njs_variables_copy(vm, &scope->variables, vm->variables_hash); if (njs_slow_path(ret != NJS_OK)) { return ret; } diff --git a/src/njs_generator.c b/src/njs_generator.c index 603d96b1..c41cd3ba 100644 --- a/src/njs_generator.c +++ b/src/njs_generator.c @@ -384,7 +384,8 @@ njs_generate(njs_vm_t *vm, njs_generator_t *generator, njs_parser_node_t *node) return njs_generate_inc_dec_operation(vm, generator, node, 1); case NJS_TOKEN_NULL: - case NJS_TOKEN_BOOLEAN: + case NJS_TOKEN_TRUE: + case NJS_TOKEN_FALSE: case NJS_TOKEN_NUMBER: case NJS_TOKEN_STRING: node->index = njs_value_index(vm, &node->u.value, generator->runtime); @@ -2321,9 +2322,10 @@ static njs_int_t njs_generate_function_declaration(njs_vm_t *vm, njs_generator_t *generator, njs_parser_node_t *node) { - njs_int_t ret; - njs_variable_t *var; - njs_function_lambda_t *lambda; + njs_int_t ret; + njs_variable_t *var; + njs_function_lambda_t *lambda; + const njs_lexer_entry_t *lex_entry; var = njs_variable_resolve(vm, node); if (njs_slow_path(var == NULL)) { @@ -2337,14 +2339,18 @@ njs_generate_function_declaration(njs_vm_t *vm, njs_generator_t *generator, lambda = njs_function_lambda(&var->value); - ret = njs_generate_function_scope(vm, lambda, node, - &node->u.reference.name); + lex_entry = njs_lexer_entry(node->u.reference.unique_id); + if (njs_slow_path(lex_entry == NULL)) { + return NJS_ERROR; + } + + ret = njs_generate_function_scope(vm, lambda, node, &lex_entry->name); if (njs_slow_path(ret != NJS_OK)) { return ret; } if (vm->debug != NULL) { - ret = njs_generate_function_debug(vm, &var->name, lambda, node); + ret = njs_generate_function_debug(vm, &lex_entry->name, lambda, node); } return ret; @@ -2473,15 +2479,17 @@ njs_generate_lambda_variables(njs_vm_t *vm, njs_generator_t *generator, { njs_index_t index; njs_variable_t *var; + njs_rbtree_node_t *rb_node; njs_vmcode_move_t *move; - njs_lvlhsh_each_t lhe; njs_vmcode_this_t *this; + njs_variable_node_t *var_node; njs_vmcode_arguments_t *arguments; - njs_lvlhsh_each_init(&lhe, &njs_variables_hash_proto); + rb_node = njs_rbtree_min(&node->scope->variables); - for ( ;; ) { - var = njs_lvlhsh_each(&node->scope->variables, &lhe); + while (njs_rbtree_is_there_successor(&node->scope->variables, rb_node)) { + var_node = (njs_variable_node_t *) rb_node; + var = var_node->variable; if (var == NULL) { break; @@ -2504,6 +2512,8 @@ njs_generate_lambda_variables(njs_vm_t *vm, njs_generator_t *generator, NJS_VMCODE_ARGUMENTS, 1); arguments->dst = var->index; } + + rb_node = njs_rbtree_node_successor(&node->scope->variables, rb_node); } return NJS_OK; @@ -3297,11 +3307,11 @@ static njs_int_t njs_generate_global_reference(njs_vm_t *vm, njs_generator_t *generator, njs_parser_node_t *node, njs_bool_t exception) { - njs_str_t *name; - njs_int_t ret; - njs_index_t index; - njs_value_t property; - njs_vmcode_prop_get_t *prop_get; + njs_int_t ret; + njs_index_t index; + njs_value_t property; + njs_vmcode_prop_get_t *prop_get; + const njs_lexer_entry_t *lex_entry; index = njs_generate_dest_index(vm, generator, node); if (njs_slow_path(index == NJS_INDEX_ERROR)) { @@ -3314,11 +3324,13 @@ njs_generate_global_reference(njs_vm_t *vm, njs_generator_t *generator, prop_get->value = index; prop_get->object = NJS_INDEX_GLOBAL_OBJECT; - /* FIXME: cache keys in a hash. */ - - name = &node->u.reference.name; + lex_entry = njs_lexer_entry(node->u.reference.unique_id); + if (njs_slow_path(lex_entry == NULL)) { + return NJS_ERROR; + } - ret = njs_string_set(vm, &property, name->start, name->length); + ret = njs_string_set(vm, &property, lex_entry->name.start, + lex_entry->name.length); if (njs_slow_path(ret != NJS_OK)) { return NJS_ERROR; } @@ -3343,6 +3355,7 @@ njs_generate_reference_error(njs_vm_t *vm, njs_generator_t *generator, njs_parser_node_t *node) { njs_jump_off_t ret; + const njs_lexer_entry_t *lex_entry; njs_vmcode_reference_error_t *ref_err; if (njs_slow_path(!node->u.reference.not_defined)) { @@ -3365,7 +3378,12 @@ njs_generate_reference_error(njs_vm_t *vm, njs_generator_t *generator, } } - return njs_name_copy(vm, &ref_err->name, &node->u.reference.name); + lex_entry = njs_lexer_entry(node->u.reference.unique_id); + if (njs_slow_path(lex_entry == NULL)) { + return NJS_ERROR; + } + + return njs_name_copy(vm, &ref_err->name, &lex_entry->name); } diff --git a/src/njs_lexer.c b/src/njs_lexer.c index 5bd19abe..46eeb7f8 100644 --- a/src/njs_lexer.c +++ b/src/njs_lexer.c @@ -18,23 +18,28 @@ struct njs_lexer_multi_s { }; +static njs_int_t njs_lexer_hash_test(njs_lvlhsh_query_t *lhq, void *data); +static njs_int_t njs_lexer_word(njs_lexer_t *lexer, njs_lexer_token_t *token); +static void njs_lexer_string(njs_lexer_t *lexer, njs_lexer_token_t *token, + u_char quote); +static void njs_lexer_number(njs_lexer_t *lexer, njs_lexer_token_t *token); +static void njs_lexer_multi(njs_lexer_t *lexer, njs_lexer_token_t *token, + const njs_lexer_multi_t *multi, size_t length); +static void njs_lexer_division(njs_lexer_t *lexer, njs_lexer_token_t *token); + static njs_lexer_token_t *njs_lexer_token_push(njs_vm_t *vm, njs_lexer_t *lexer); static njs_lexer_token_t *njs_lexer_token_pop(njs_lexer_t *lexer); -static njs_token_t njs_lexer_token_name_resolve(njs_lexer_t *lexer, - njs_lexer_token_t *lt); -static njs_token_t njs_lexer_next_token(njs_lexer_t *lexer, - njs_lexer_token_t *lt); -static njs_token_t njs_lexer_word(njs_lexer_t *lexer, njs_lexer_token_t *lt, - u_char c); -static njs_token_t njs_lexer_string(njs_lexer_t *lexer, njs_lexer_token_t *lt, - u_char quote); -static njs_token_t njs_lexer_number(njs_lexer_t *lexer, njs_lexer_token_t *lt, - u_char c); -static njs_token_t njs_lexer_multi(njs_lexer_t *lexer, njs_lexer_token_t *lt, - njs_token_t token, njs_uint_t n, const njs_lexer_multi_t *multi); -static njs_token_t njs_lexer_division(njs_lexer_t *lexer, - njs_token_t token); + + +const njs_lvlhsh_proto_t njs_lexer_hash_proto + njs_aligned(64) = +{ + NJS_LVLHSH_DEFAULT, + njs_lexer_hash_test, + njs_lvlhsh_alloc, + njs_lvlhsh_free, +}; static const uint8_t njs_tokens[256] njs_aligned(64) = { @@ -297,7 +302,8 @@ njs_lexer_init(njs_vm_t *vm, njs_lexer_t *lexer, njs_str_t *file, lexer->start = start; lexer->end = end; lexer->line = 1; - lexer->keywords_hash = vm->shared->keywords_hash; + lexer->keywords_hash = &vm->shared->keywords_hash; + lexer->mem_pool = vm->mem_pool; njs_queue_init(&lexer->preread); @@ -312,9 +318,9 @@ njs_lexer_token(njs_vm_t *vm, njs_lexer_t *lexer) lexer->prev_start = lexer->start; - if (lexer->lexer_token != NULL) { - lexer->prev_token = lexer->lexer_token->token; - njs_mp_free(vm->mem_pool, lexer->lexer_token); + if (lexer->token != NULL) { + lexer->prev_token = lexer->token->type; + njs_mp_free(vm->mem_pool, lexer->token); } if (njs_queue_is_empty(&lexer->preread)) { @@ -324,9 +330,9 @@ njs_lexer_token(njs_vm_t *vm, njs_lexer_t *lexer) } } - lexer->lexer_token = njs_lexer_token_pop(lexer); + lexer->token = njs_lexer_token_pop(lexer); - return njs_lexer_token_name_resolve(lexer, lexer->lexer_token); + return lexer->token->type; } @@ -350,9 +356,7 @@ njs_lexer_peek_token(njs_vm_t *vm, njs_lexer_t *lexer, size_t offset) /* NJS_TOKEN_DIVISION stands for regexp literal. */ - if (lt->token == NJS_TOKEN_DIVISION - || lt->token == NJS_TOKEN_END) - { + if (lt->type == NJS_TOKEN_DIVISION || lt->type == NJS_TOKEN_END) { break; } @@ -368,232 +372,283 @@ njs_lexer_peek_token(njs_vm_t *vm, njs_lexer_t *lexer, size_t offset) } } - return njs_lexer_token_name_resolve(lexer, lt); + return lt->type; } -static njs_lexer_token_t * -njs_lexer_token_push(njs_vm_t *vm, njs_lexer_t *lexer) +njs_int_t +njs_lexer_rollback(njs_vm_t *vm, njs_lexer_t *lexer) { njs_lexer_token_t *lt; lt = njs_mp_zalloc(vm->mem_pool, sizeof(njs_lexer_token_t)); if (njs_slow_path(lt == NULL)) { - return NULL; + return NJS_ERROR; } - lt->token = njs_lexer_next_token(lexer, lt); + *lt = *lexer->token; - njs_queue_insert_tail(&lexer->preread, <->link); + njs_queue_insert_head(&lexer->preread, <->link); - return lt; + return NJS_OK; } static njs_lexer_token_t * -njs_lexer_token_pop(njs_lexer_t *lexer) -{ - njs_queue_link_t *lnk; - - lnk = njs_queue_first(&lexer->preread); - njs_queue_remove(lnk); - - return njs_queue_link_data(lnk, njs_lexer_token_t, link); -} - - -njs_int_t -njs_lexer_rollback(njs_vm_t *vm, njs_lexer_t *lexer) +njs_lexer_token_push(njs_vm_t *vm, njs_lexer_t *lexer) { - njs_lexer_token_t *lt; + njs_int_t ret; + njs_lexer_token_t *token; - lt = njs_mp_zalloc(vm->mem_pool, sizeof(njs_lexer_token_t)); - if (njs_slow_path(lt == NULL)) { - return NJS_ERROR; + token = njs_mp_zalloc(vm->mem_pool, sizeof(njs_lexer_token_t)); + if (njs_slow_path(token == NULL)) { + return NULL; } - *lt = *lexer->lexer_token; + do { + ret = njs_lexer_next_token(lexer, token); + if (njs_slow_path(ret != NJS_OK)) { + return NULL; + } + + } while (token->type == NJS_TOKEN_COMMENT); - njs_queue_insert_head(&lexer->preread, <->link); + njs_queue_insert_tail(&lexer->preread, &token->link); - return NJS_OK; + return token; } -static njs_token_t -njs_lexer_token_name_resolve(njs_lexer_t *lexer, njs_lexer_token_t *lt) +static njs_lexer_token_t * +njs_lexer_token_pop(njs_lexer_t *lexer) { - if (lt->token == NJS_TOKEN_NAME) { - njs_lexer_keyword(lexer, lt); - } + njs_queue_link_t *lnk; + + lnk = njs_queue_first(&lexer->preread); + njs_queue_remove(lnk); - return lt->token; + return njs_queue_link_data(lnk, njs_lexer_token_t, link); } -static njs_token_t -njs_lexer_next_token(njs_lexer_t *lexer, njs_lexer_token_t *lt) +njs_int_t +njs_lexer_next_token(njs_lexer_t *lexer, njs_lexer_token_t *token) { - u_char c, *p; - njs_uint_t n; - njs_token_t token; - const njs_lexer_multi_t *multi; + u_char c, *p; - lt->text.start = lexer->start; + c = ' '; while (lexer->start < lexer->end) { c = *lexer->start++; - token = njs_tokens[c]; - - switch (token) { - - case NJS_TOKEN_SPACE: - lt->text.start = lexer->start; - continue; - - case NJS_TOKEN_LETTER: - return njs_lexer_word(lexer, lt, c); - - case NJS_TOKEN_DOUBLE_QUOTE: - case NJS_TOKEN_SINGLE_QUOTE: - return njs_lexer_string(lexer, lt, c); - - case NJS_TOKEN_DOT: - p = lexer->start; - - if (p + 1 < lexer->end - && njs_tokens[p[0]] == NJS_TOKEN_DOT - && njs_tokens[p[1]] == NJS_TOKEN_DOT) - { - lt->text.length = (p - lt->text.start) + 2; - lexer->start += 2; - return NJS_TOKEN_ELLIPSIS; - } - - if (p == lexer->end || njs_tokens[*p] != NJS_TOKEN_DIGIT) { - lt->text.length = p - lt->text.start; - return NJS_TOKEN_DOT; - } - - /* Fall through. */ - - case NJS_TOKEN_DIGIT: - return njs_lexer_number(lexer, lt, c); - - case NJS_TOKEN_ASSIGNMENT: - n = njs_nitems(njs_assignment_token), - multi = njs_assignment_token; - - goto multi; + if (njs_tokens[c] != NJS_TOKEN_SPACE) { + break; + } + } - case NJS_TOKEN_ADDITION: - n = njs_nitems(njs_addition_token), - multi = njs_addition_token; + lexer->keyword = 0; + token->type = njs_tokens[c]; - goto multi; + switch (token->type) { - case NJS_TOKEN_SUBSTRACTION: - n = njs_nitems(njs_substraction_token), - multi = njs_substraction_token; + case NJS_TOKEN_LETTER: + return njs_lexer_word(lexer, token); - goto multi; + case NJS_TOKEN_DOUBLE_QUOTE: + case NJS_TOKEN_SINGLE_QUOTE: + njs_lexer_string(lexer, token, c); + break; - case NJS_TOKEN_MULTIPLICATION: - n = njs_nitems(njs_multiplication_token), - multi = njs_multiplication_token; + case NJS_TOKEN_DOT: + p = lexer->start; - goto multi; + if (p + 1 < lexer->end + && njs_tokens[p[0]] == NJS_TOKEN_DOT + && njs_tokens[p[1]] == NJS_TOKEN_DOT) + { + token->text.start = lexer->start - 1; + token->text.length = (p - token->text.start) + 2; - case NJS_TOKEN_DIVISION: - token = njs_lexer_division(lexer, token); + token->type = NJS_TOKEN_ELLIPSIS; - if (token != NJS_TOKEN_AGAIN) { - goto done; - } + lexer->start += 2; - continue; + return NJS_OK; + } - case NJS_TOKEN_REMAINDER: - n = njs_nitems(njs_remainder_token), - multi = njs_remainder_token; + if (p == lexer->end || njs_tokens[*p] != NJS_TOKEN_DIGIT) { + token->text.start = lexer->start - 1; + token->text.length = p - token->text.start; - goto multi; + token->type = NJS_TOKEN_DOT; - case NJS_TOKEN_BITWISE_AND: - n = njs_nitems(njs_bitwise_and_token), - multi = njs_bitwise_and_token; + return NJS_OK; + } - goto multi; + /* Fall through. */ + + case NJS_TOKEN_DIGIT: + njs_lexer_number(lexer, token); + break; + + case NJS_TOKEN_DIVISION: + njs_lexer_division(lexer, token); + break; + + case NJS_TOKEN_ASSIGNMENT: + njs_lexer_multi(lexer, token, njs_assignment_token, + njs_nitems(njs_assignment_token)); + break; + + case NJS_TOKEN_ADDITION: + njs_lexer_multi(lexer, token, njs_addition_token, + njs_nitems(njs_addition_token)); + break; + + case NJS_TOKEN_SUBSTRACTION: + njs_lexer_multi(lexer, token, njs_substraction_token, + njs_nitems(njs_substraction_token)); + break; + + case NJS_TOKEN_MULTIPLICATION: + njs_lexer_multi(lexer, token, njs_multiplication_token, + njs_nitems(njs_multiplication_token)); + break; + + case NJS_TOKEN_REMAINDER: + njs_lexer_multi(lexer, token, njs_remainder_token, + njs_nitems(njs_remainder_token)); + break; + + case NJS_TOKEN_BITWISE_AND: + njs_lexer_multi(lexer, token, njs_bitwise_and_token, + njs_nitems(njs_bitwise_and_token)); + break; + + case NJS_TOKEN_BITWISE_XOR: + njs_lexer_multi(lexer, token, njs_bitwise_xor_token, + njs_nitems(njs_bitwise_xor_token)); + break; + + case NJS_TOKEN_BITWISE_OR: + njs_lexer_multi(lexer, token, njs_bitwise_or_token, + njs_nitems(njs_bitwise_or_token)); + break; + + case NJS_TOKEN_LOGICAL_NOT: + njs_lexer_multi(lexer, token, njs_logical_not_token, + njs_nitems(njs_logical_not_token)); + break; + + case NJS_TOKEN_LESS: + njs_lexer_multi(lexer, token, njs_less_token, + njs_nitems(njs_less_token)); + break; + + case NJS_TOKEN_GREATER: + njs_lexer_multi(lexer, token, njs_greater_token, + njs_nitems(njs_greater_token)); + break; + + case NJS_TOKEN_CONDITIONAL: + njs_lexer_multi(lexer, token, njs_conditional_token, + njs_nitems(njs_conditional_token)); + break; + + case NJS_TOKEN_SPACE: + token->type = NJS_TOKEN_END; + return NJS_OK; + + case NJS_TOKEN_LINE_END: + lexer->line++; + + /* Fall through. */ + + default: + token->text.start = lexer->start - 1; + token->text.length = lexer->start - token->text.start; + + break; + } - case NJS_TOKEN_BITWISE_XOR: - n = njs_nitems(njs_bitwise_xor_token), - multi = njs_bitwise_xor_token; + return NJS_OK; +} - goto multi; - case NJS_TOKEN_BITWISE_OR: - n = njs_nitems(njs_bitwise_or_token), - multi = njs_bitwise_or_token; +static njs_int_t +njs_lexer_hash_test(njs_lvlhsh_query_t *lhq, void *data) +{ + njs_lexer_entry_t *entry; - goto multi; + entry = data; - case NJS_TOKEN_LOGICAL_NOT: - n = njs_nitems(njs_logical_not_token), - multi = njs_logical_not_token; + if (entry->name.length == lhq->key.length + && memcmp(entry->name.start, lhq->key.start, lhq->key.length) == 0) + { + return NJS_OK; + } - goto multi; + return NJS_DECLINED; +} - case NJS_TOKEN_LESS: - n = njs_nitems(njs_less_token), - multi = njs_less_token; - goto multi; +static njs_lexer_entry_t * +njs_lexer_keyword_find(njs_lexer_t *lexer, u_char *key, size_t length, + uint32_t hash) +{ + njs_int_t ret; + njs_lexer_entry_t *entry; + njs_lvlhsh_query_t lhq; - case NJS_TOKEN_GREATER: - n = njs_nitems(njs_greater_token), - multi = njs_greater_token; + lhq.key.start = key; + lhq.key.length = length; - goto multi; + lhq.key_hash = hash; + lhq.proto = &njs_lexer_hash_proto; - case NJS_TOKEN_CONDITIONAL: - n = njs_nitems(njs_conditional_token), - multi = njs_conditional_token; + ret = njs_lvlhsh_find(lexer->keywords_hash, &lhq); + if (ret == NJS_OK) { + return lhq.value; + } - goto multi; + entry = njs_mp_alloc(lexer->mem_pool, sizeof(njs_lexer_entry_t)); + if (njs_slow_path(entry == NULL)) { + return NULL; + } - case NJS_TOKEN_LINE_END: - lexer->line++; + entry->name.start = njs_mp_alloc(lexer->mem_pool, length + 1); + if (njs_slow_path(entry->name.start == NULL)) { + return NULL; + } - /* Fall through. */ + memcpy(entry->name.start, key, length); - default: - goto done; - } + entry->name.start[length] = '\0'; + entry->name.length = length; - multi: + lhq.value = entry; + lhq.pool = lexer->mem_pool; - return njs_lexer_multi(lexer, lt, token, n, multi); + ret = njs_lvlhsh_insert(lexer->keywords_hash, &lhq); + if (njs_slow_path(ret != NJS_OK)) { + return NULL; } - token = NJS_TOKEN_END; - -done: - - lt->text.length = lexer->start - lt->text.start; - - return token; + return entry; } -static njs_token_t -njs_lexer_word(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char c) +static njs_int_t +njs_lexer_word(njs_lexer_t *lexer, njs_lexer_token_t *token) { - u_char *p; + u_char *p, c; + uint32_t hash_id; + const njs_lexer_entry_t *entry; + const njs_lexer_keyword_entry_t *key_entry; /* TODO: UTF-8 */ - static const uint8_t letter_digit[32] njs_aligned(32) = { + static const uint8_t letter_digit[32] njs_aligned(32) = { 0x00, 0x00, 0x00, 0x00, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ /* '&%$ #"! /.-, |*)( 7654 3210 ?>=< ;:98 */ @@ -611,9 +666,10 @@ njs_lexer_word(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char c) 0x00, 0x00, 0x00, 0x00, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ }; - lt->token_line = lexer->line; - lt->key_hash = njs_djb_hash_add(NJS_DJB_HASH_INIT, c); - lt->text.start = lexer->start - 1; + token->line = lexer->line; + token->text.start = lexer->start - 1; + + hash_id = njs_djb_hash_add(NJS_DJB_HASH_INIT, *token->text.start); for (p = lexer->start; p < lexer->end; p++) { c = *p; @@ -622,25 +678,46 @@ njs_lexer_word(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char c) break; } - lt->key_hash = njs_djb_hash_add(lt->key_hash, c); + hash_id = njs_djb_hash_add(hash_id, c); } + token->text.length = p - token->text.start; lexer->start = p; - lt->text.length = p - lt->text.start; - return NJS_TOKEN_NAME; + key_entry = njs_lexer_keyword(token->text.start, token->text.length); + + if (key_entry == NULL) { + entry = njs_lexer_keyword_find(lexer, token->text.start, + token->text.length, hash_id); + if (njs_slow_path(entry == NULL)) { + return NJS_ERROR; + } + + token->type = NJS_TOKEN_NAME; + + } else { + entry = &key_entry->value->entry; + token->type = key_entry->value->type; + + lexer->keyword = 1; + } + + token->unique_id = (uintptr_t) entry; + + return NJS_OK; } -static njs_token_t -njs_lexer_string(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char quote) +static void +njs_lexer_string(njs_lexer_t *lexer, njs_lexer_token_t *token, u_char quote) { u_char *p, c; njs_bool_t escape; escape = 0; - lt->text.start = lexer->start; + p = lexer->start; + token->text.start = p; while (p < lexer->end) { @@ -670,32 +747,32 @@ njs_lexer_string(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char quote) if (c == quote) { lexer->start = p; - lt->text.length = (p - 1) - lt->text.start; - - if (escape == 0) { - return NJS_TOKEN_STRING; - } + token->text.length = (p - 1) - token->text.start; - return NJS_TOKEN_ESCAPE_STRING; + token->type = (escape == 0) ? NJS_TOKEN_STRING + : NJS_TOKEN_ESCAPE_STRING; + return; } } - lt->text.start--; - lt->text.length = p - lt->text.start; + token->text.start--; + token->text.length = p - token->text.start; - return NJS_TOKEN_UNTERMINATED_STRING; + token->type = NJS_TOKEN_UNTERMINATED_STRING; } -static njs_token_t -njs_lexer_number(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char c) +static void +njs_lexer_number(njs_lexer_t *lexer, njs_lexer_token_t *token) { + u_char c; const u_char *p; - lt->text.start = lexer->start - 1; - + c = lexer->start[-1]; p = lexer->start; + token->text.start = lexer->start - 1; + if (c == '0' && p != lexer->end) { /* Hexadecimal literal values. */ @@ -707,7 +784,7 @@ njs_lexer_number(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char c) goto illegal_token; } - lt->number = njs_number_hex_parse(&p, lexer->end); + token->number = njs_number_hex_parse(&p, lexer->end); goto done; } @@ -721,7 +798,7 @@ njs_lexer_number(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char c) goto illegal_token; } - lt->number = njs_number_oct_parse(&p, lexer->end); + token->number = njs_number_oct_parse(&p, lexer->end); if (p < lexer->end && (*p == '8' || *p == '9')) { goto illegal_trailer; @@ -739,7 +816,7 @@ njs_lexer_number(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char c) goto illegal_token; } - lt->number = njs_number_bin_parse(&p, lexer->end); + token->number = njs_number_bin_parse(&p, lexer->end); if (p < lexer->end && (*p >= '2' && *p <= '9')) { goto illegal_trailer; @@ -756,14 +833,16 @@ njs_lexer_number(njs_lexer_t *lexer, njs_lexer_token_t *lt, u_char c) } p--; - lt->number = njs_number_dec_parse(&p, lexer->end); + token->number = njs_number_dec_parse(&p, lexer->end); done: lexer->start = (u_char *) p; - lt->text.length = p - lt->text.start; + token->text.length = p - token->text.start; + + token->type = NJS_TOKEN_NUMBER; - return NJS_TOKEN_NUMBER; + return; illegal_trailer: @@ -771,92 +850,105 @@ illegal_trailer: illegal_token: - lt->text.length = p - lt->text.start; + token->text.length = p - token->text.start; - return NJS_TOKEN_ILLEGAL; + token->type = NJS_TOKEN_ILLEGAL; } -static njs_token_t -njs_lexer_multi(njs_lexer_t *lexer, njs_lexer_token_t *lt, njs_token_t token, - njs_uint_t n, const njs_lexer_multi_t *multi) +static void +njs_lexer_multi(njs_lexer_t *lexer, njs_lexer_token_t *token, + const njs_lexer_multi_t *multi, size_t length) { u_char c; - if (lexer->start < lexer->end) { + token->text.start = lexer->start - 1; + + while (length != 0 && multi != NULL) { c = lexer->start[0]; - do { - if (c == multi->symbol) { - lexer->start++; + if (c == multi->symbol) { + lexer->start++; - if (multi->count == 0) { - token = multi->token; - break; - } + token->type = multi->token; - return njs_lexer_multi(lexer, lt, multi->token, multi->count, - multi->next); + if (multi->count == 0) { + break; } - multi++; - n--; + length = multi->count; + multi = multi->next; - } while (n != 0); + } else { + length--; + multi++; + } } - lt->text.length = lexer->start - lt->text.start; - - return token; + token->text.length = lexer->start - token->text.start; } -static njs_token_t -njs_lexer_division(njs_lexer_t *lexer, njs_token_t token) +static void +njs_lexer_division(njs_lexer_t *lexer, njs_lexer_token_t *token) { u_char c, *p; - if (lexer->start < lexer->end) { - c = lexer->start[0]; + token->text.start = lexer->start - 1; - if (c == '/') { - token = NJS_TOKEN_END; - lexer->start++; + if (lexer->start >= lexer->end) { + goto done; + } - for (p = lexer->start; p < lexer->end; p++) { + c = lexer->start[0]; - if (*p == '\n') { - lexer->start = p + 1; - lexer->line++; - return NJS_TOKEN_LINE_END; - } + if (c == '/') { + token->type = NJS_TOKEN_END; + + lexer->start++; + + for (p = lexer->start; p < lexer->end; p++) { + + if (*p == '\n') { + lexer->start = p + 1; + lexer->line++; + + token->type = NJS_TOKEN_LINE_END; + + goto done; } + } - } else if (c == '*') { - lexer->start++; + } else if (c == '*') { + lexer->start++; - for (p = lexer->start; p < lexer->end; p++) { + for (p = lexer->start; p < lexer->end; p++) { - if (*p == '\n') { - lexer->line++; - continue; - } + if (*p == '\n') { + lexer->line++; + continue; + } + + if (*p == '*') { + if (p + 1 < lexer->end && p[1] == '/') { + lexer->start = p + 2; - if (*p == '*') { - if (p + 1 < lexer->end && p[1] == '/') { - lexer->start = p + 2; - return NJS_TOKEN_AGAIN; - } + token->type = NJS_TOKEN_COMMENT; + + goto done; } } + } - return NJS_TOKEN_ILLEGAL; + token->type = NJS_TOKEN_ILLEGAL; - } else if (c == '=') { - lexer->start++; - token = NJS_TOKEN_DIVISION_ASSIGNMENT; - } + } else if (c == '=') { + lexer->start++; + + token->type = NJS_TOKEN_DIVISION_ASSIGNMENT; } - return token; +done: + + token->text.length = lexer->start - token->text.start; } diff --git a/src/njs_lexer.h b/src/njs_lexer.h index 8c3abb82..ad037ac0 100644 --- a/src/njs_lexer.h +++ b/src/njs_lexer.h @@ -9,7 +9,6 @@ typedef enum { - NJS_TOKEN_AGAIN = -2, NJS_TOKEN_ERROR = -1, NJS_TOKEN_ILLEGAL = 0, @@ -35,6 +34,8 @@ typedef enum { NJS_TOKEN_COLON, NJS_TOKEN_CONDITIONAL, + NJS_TOKEN_COMMENT, + NJS_TOKEN_ASSIGNMENT, NJS_TOKEN_ARROW, NJS_TOKEN_ADDITION_ASSIGNMENT, @@ -112,7 +113,8 @@ typedef enum { NJS_TOKEN_NULL, NJS_TOKEN_NUMBER, - NJS_TOKEN_BOOLEAN, + NJS_TOKEN_TRUE, + NJS_TOKEN_FALSE, NJS_TOKEN_STRING, #define NJS_TOKEN_LAST_CONST NJS_TOKEN_STRING @@ -176,14 +178,50 @@ typedef enum { NJS_TOKEN_IMPORT, NJS_TOKEN_EXPORT, + NJS_TOKEN_AWAIT, + NJS_TOKEN_CLASS, + NJS_TOKEN_CONST, + NJS_TOKEN_DEBUGGER, + NJS_TOKEN_ENUM, + NJS_TOKEN_EXTENDS, + NJS_TOKEN_IMPLEMENTS, + NJS_TOKEN_INTERFACE, + NJS_TOKEN_LET, + NJS_TOKEN_PACKAGE, + NJS_TOKEN_PRIVATE, + NJS_TOKEN_PROTECTED, + NJS_TOKEN_PUBLIC, + NJS_TOKEN_STATIC, + NJS_TOKEN_SUPER, + NJS_TOKEN_RESERVED, } njs_token_t; typedef struct { - njs_token_t token:16; - uint32_t token_line; - uint32_t key_hash; + njs_str_t name; +} njs_lexer_entry_t; + + +typedef struct { + njs_lexer_entry_t entry; + njs_token_t type; +} njs_keyword_t; + + +typedef struct { + const char *key; + const njs_keyword_t *value; + + size_t length; + size_t next; +} njs_lexer_keyword_entry_t; + + +typedef struct { + njs_token_t type:16; + uint32_t line; + uintptr_t unique_id; njs_str_t text; double number; njs_queue_link_t link; @@ -191,7 +229,7 @@ typedef struct { typedef struct { - njs_lexer_token_t *lexer_token; + njs_lexer_token_t *token; njs_queue_t preread; /* of njs_lexer_token_t */ uint8_t keyword; @@ -201,28 +239,38 @@ typedef struct { uint32_t line; njs_str_t file; - njs_lvlhsh_t keywords_hash; + njs_lvlhsh_t *keywords_hash; + + njs_mp_t *mem_pool; u_char *start; u_char *end; } njs_lexer_t; -typedef struct { - njs_str_t name; - njs_token_t token; - double number; -} njs_keyword_t; - - njs_int_t njs_lexer_init(njs_vm_t *vm, njs_lexer_t *lexer, njs_str_t *file, u_char *start, u_char *end); + njs_token_t njs_lexer_token(njs_vm_t *vm, njs_lexer_t *lexer); njs_token_t njs_lexer_peek_token(njs_vm_t *vm, njs_lexer_t *lexer, size_t offset); njs_int_t njs_lexer_rollback(njs_vm_t *vm, njs_lexer_t *lexer); -njs_int_t njs_lexer_keywords_init(njs_mp_t *mp, njs_lvlhsh_t *hash); -void njs_lexer_keyword(njs_lexer_t *lexer, njs_lexer_token_t *lt); + +njs_int_t njs_lexer_next_token(njs_lexer_t *lexer, njs_lexer_token_t *token); + +const njs_lexer_keyword_entry_t *njs_lexer_keyword(const u_char *key, + size_t length); +njs_int_t njs_lexer_keywords(njs_arr_t *array); + + +njs_inline const njs_lexer_entry_t * +njs_lexer_entry(uintptr_t unique_id) +{ + return (const njs_lexer_entry_t *) unique_id; +} + + +extern const njs_lvlhsh_proto_t njs_lexer_hash_proto; #endif /* _NJS_LEXER_H_INCLUDED_ */ diff --git a/src/njs_lexer_keyword.c b/src/njs_lexer_keyword.c index f5b329df..310e9992 100644 --- a/src/njs_lexer_keyword.c +++ b/src/njs_lexer_keyword.c @@ -1,154 +1,76 @@ /* - * Copyright (C) Igor Sysoev * Copyright (C) NGINX, Inc. */ #include +#include -static const njs_keyword_t njs_keywords[] = { - - /* Values. */ - - { njs_str("null"), NJS_TOKEN_NULL, 0 }, - { njs_str("false"), NJS_TOKEN_BOOLEAN, 0 }, - { njs_str("true"), NJS_TOKEN_BOOLEAN, 1 }, - - /* Operators. */ - - { njs_str("in"), NJS_TOKEN_IN, 0 }, - { njs_str("typeof"), NJS_TOKEN_TYPEOF, 0 }, - { njs_str("instanceof"), NJS_TOKEN_INSTANCEOF, 0 }, - { njs_str("void"), NJS_TOKEN_VOID, 0 }, - { njs_str("new"), NJS_TOKEN_NEW, 0 }, - { njs_str("delete"), NJS_TOKEN_DELETE, 0 }, - { njs_str("yield"), NJS_TOKEN_YIELD, 0 }, - - /* Statements. */ - - { njs_str("var"), NJS_TOKEN_VAR, 0 }, - { njs_str("if"), NJS_TOKEN_IF, 0 }, - { njs_str("else"), NJS_TOKEN_ELSE, 0 }, - { njs_str("while"), NJS_TOKEN_WHILE, 0 }, - { njs_str("do"), NJS_TOKEN_DO, 0 }, - { njs_str("for"), NJS_TOKEN_FOR, 0 }, - { njs_str("break"), NJS_TOKEN_BREAK, 0 }, - { njs_str("continue"), NJS_TOKEN_CONTINUE, 0 }, - { njs_str("switch"), NJS_TOKEN_SWITCH, 0 }, - { njs_str("case"), NJS_TOKEN_CASE, 0 }, - { njs_str("default"), NJS_TOKEN_DEFAULT, 0 }, - { njs_str("function"), NJS_TOKEN_FUNCTION, 0 }, - { njs_str("return"), NJS_TOKEN_RETURN, 0 }, - { njs_str("with"), NJS_TOKEN_WITH, 0 }, - { njs_str("try"), NJS_TOKEN_TRY, 0 }, - { njs_str("catch"), NJS_TOKEN_CATCH, 0 }, - { njs_str("finally"), NJS_TOKEN_FINALLY, 0 }, - { njs_str("throw"), NJS_TOKEN_THROW, 0 }, - - /* Module. */ - - { njs_str("import"), NJS_TOKEN_IMPORT, 0 }, - { njs_str("export"), NJS_TOKEN_EXPORT, 0 }, - - /* Reserved words. */ - - { njs_str("this"), NJS_TOKEN_THIS, 0 }, - { njs_str("arguments"), NJS_TOKEN_ARGUMENTS, 0 }, - { njs_str("eval"), NJS_TOKEN_EVAL, 0 }, - - { njs_str("await"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("class"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("const"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("debugger"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("enum"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("extends"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("implements"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("interface"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("let"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("package"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("private"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("protected"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("public"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("static"), NJS_TOKEN_RESERVED, 0 }, - { njs_str("super"), NJS_TOKEN_RESERVED, 0 }, -}; - - -static njs_int_t -njs_keyword_hash_test(njs_lvlhsh_query_t *lhq, void *data) +njs_inline int +njs_lexer_keyword_hash(const u_char *key, size_t size, size_t table_size) { - njs_keyword_t *keyword; - - keyword = data; - - if (njs_strstr_eq(&lhq->key, &keyword->name)) { - return NJS_OK; - } - - return NJS_DECLINED; + return ((((key[0] * key[size - 1]) + size) % table_size) + 0x01); } -const njs_lvlhsh_proto_t njs_keyword_hash_proto - njs_aligned(64) = +njs_inline const njs_lexer_keyword_entry_t * +njs_lexer_keyword_entry(const njs_lexer_keyword_entry_t *root, + const u_char *key, size_t length) { - NJS_LVLHSH_DEFAULT, - njs_keyword_hash_test, - njs_lvlhsh_alloc, - njs_lvlhsh_free, -}; + const njs_lexer_keyword_entry_t *entry; + entry = root + njs_lexer_keyword_hash(key, length, root->length); -njs_int_t -njs_lexer_keywords_init(njs_mp_t *mp, njs_lvlhsh_t *hash) -{ - njs_uint_t n; - njs_lvlhsh_query_t lhq; - const njs_keyword_t *keyword; + while (entry->key != NULL) { + if (entry->length == length) { + if (strncmp(entry->key, (char *) key, length) == 0) { + return entry; + } - keyword = njs_keywords; - n = njs_nitems(njs_keywords); + entry = &root[entry->next]; - lhq.replace = 0; - lhq.proto = &njs_keyword_hash_proto; - lhq.pool = mp; + } else if (entry->length > length) { + return NULL; - do { - lhq.key_hash = njs_djb_hash(keyword->name.start, keyword->name.length); - lhq.key = keyword->name; - lhq.value = (void *) keyword; - - if (njs_slow_path(njs_lvlhsh_insert(hash, &lhq) != NJS_OK)) { - return NJS_ERROR; + } else { + entry = &root[entry->next]; } + } - keyword++; - n--; + return NULL; +} - } while (n != 0); - return NJS_OK; +const njs_lexer_keyword_entry_t * +njs_lexer_keyword(const u_char *key, size_t length) +{ + const njs_lexer_keyword_entry_t *entry; + + entry = njs_lexer_keyword_entry(njs_lexer_keyword_entries, key, length); + if (njs_slow_path(entry == NULL)) { + return NULL; + } + + return entry; } -void -njs_lexer_keyword(njs_lexer_t *lexer, njs_lexer_token_t *lt) +njs_int_t +njs_lexer_keywords(njs_arr_t *list) { - njs_keyword_t *keyword; - njs_lvlhsh_query_t lhq; + njs_str_t *kw; + njs_uint_t i; - lhq.key_hash = lt->key_hash; - lhq.key = lt->text; - lhq.proto = &njs_keyword_hash_proto; - - lexer->keyword = 0; + for (i = 0; i < sizeof(njs_lexer_kws) / sizeof(njs_keyword_t); i++) { + kw = njs_arr_add(list); + if (njs_slow_path(kw == NULL)) { + return NJS_ERROR; + } - if (njs_lvlhsh_find(&lexer->keywords_hash, &lhq) == NJS_OK) { - keyword = lhq.value; - lt->token = keyword->token; - lt->number = keyword->number; - lexer->keyword = 1; + *kw = njs_lexer_kws[i].entry.name; } + + return NJS_OK; } diff --git a/src/njs_lexer_tables.h b/src/njs_lexer_tables.h new file mode 100644 index 00000000..8e1ff013 --- /dev/null +++ b/src/njs_lexer_tables.h @@ -0,0 +1,146 @@ + +/* + * Copyright (C) Nginx, Inc. + * + * Do not edit, generated by: utils/lexer_keyword.py. + */ + + +#ifndef _NJS_LEXER_TABLES_H_INCLUDED_ +#define _NJS_LEXER_TABLES_H_INCLUDED_ + + +static const njs_keyword_t njs_lexer_kws[48] = +{ + { .entry = { njs_str("null") }, .type = NJS_TOKEN_NULL }, + { .entry = { njs_str("false") }, .type = NJS_TOKEN_FALSE }, + { .entry = { njs_str("true") }, .type = NJS_TOKEN_TRUE }, + { .entry = { njs_str("in") }, .type = NJS_TOKEN_IN }, + { .entry = { njs_str("typeof") }, .type = NJS_TOKEN_TYPEOF }, + { .entry = { njs_str("instanceof") }, .type = NJS_TOKEN_INSTANCEOF }, + { .entry = { njs_str("void") }, .type = NJS_TOKEN_VOID }, + { .entry = { njs_str("new") }, .type = NJS_TOKEN_NEW }, + { .entry = { njs_str("delete") }, .type = NJS_TOKEN_DELETE }, + { .entry = { njs_str("yield") }, .type = NJS_TOKEN_YIELD }, + { .entry = { njs_str("var") }, .type = NJS_TOKEN_VAR }, + { .entry = { njs_str("if") }, .type = NJS_TOKEN_IF }, + { .entry = { njs_str("else") }, .type = NJS_TOKEN_ELSE }, + { .entry = { njs_str("while") }, .type = NJS_TOKEN_WHILE }, + { .entry = { njs_str("do") }, .type = NJS_TOKEN_DO }, + { .entry = { njs_str("for") }, .type = NJS_TOKEN_FOR }, + { .entry = { njs_str("break") }, .type = NJS_TOKEN_BREAK }, + { .entry = { njs_str("continue") }, .type = NJS_TOKEN_CONTINUE }, + { .entry = { njs_str("switch") }, .type = NJS_TOKEN_SWITCH }, + { .entry = { njs_str("case") }, .type = NJS_TOKEN_CASE }, + { .entry = { njs_str("default") }, .type = NJS_TOKEN_DEFAULT }, + { .entry = { njs_str("function") }, .type = NJS_TOKEN_FUNCTION }, + { .entry = { njs_str("return") }, .type = NJS_TOKEN_RETURN }, + { .entry = { njs_str("with") }, .type = NJS_TOKEN_WITH }, + { .entry = { njs_str("try") }, .type = NJS_TOKEN_TRY }, + { .entry = { njs_str("catch") }, .type = NJS_TOKEN_CATCH }, + { .entry = { njs_str("finally") }, .type = NJS_TOKEN_FINALLY }, + { .entry = { njs_str("throw") }, .type = NJS_TOKEN_THROW }, + { .entry = { njs_str("import") }, .type = NJS_TOKEN_IMPORT }, + { .entry = { njs_str("export") }, .type = NJS_TOKEN_EXPORT }, + { .entry = { njs_str("this") }, .type = NJS_TOKEN_THIS }, + { .entry = { njs_str("arguments") }, .type = NJS_TOKEN_ARGUMENTS }, + { .entry = { njs_str("eval") }, .type = NJS_TOKEN_EVAL }, + { .entry = { njs_str("await") }, .type = NJS_TOKEN_AWAIT }, + { .entry = { njs_str("class") }, .type = NJS_TOKEN_CLASS }, + { .entry = { njs_str("const") }, .type = NJS_TOKEN_CONST }, + { .entry = { njs_str("debugger") }, .type = NJS_TOKEN_DEBUGGER }, + { .entry = { njs_str("enum") }, .type = NJS_TOKEN_ENUM }, + { .entry = { njs_str("extends") }, .type = NJS_TOKEN_EXTENDS }, + { .entry = { njs_str("implements") }, .type = NJS_TOKEN_IMPLEMENTS }, + { .entry = { njs_str("interface") }, .type = NJS_TOKEN_INTERFACE }, + { .entry = { njs_str("let") }, .type = NJS_TOKEN_LET }, + { .entry = { njs_str("package") }, .type = NJS_TOKEN_PACKAGE }, + { .entry = { njs_str("private") }, .type = NJS_TOKEN_PRIVATE }, + { .entry = { njs_str("protected") }, .type = NJS_TOKEN_PROTECTED }, + { .entry = { njs_str("public") }, .type = NJS_TOKEN_PUBLIC }, + { .entry = { njs_str("static") }, .type = NJS_TOKEN_STATIC }, + { .entry = { njs_str("super") }, .type = NJS_TOKEN_SUPER }, +}; + + +static const njs_lexer_keyword_entry_t njs_lexer_keyword_entries[75] = +{ + { NULL, NULL, 74, 0 }, + { "case", &njs_lexer_kws[19], 4, 0 }, + { "continue", &njs_lexer_kws[17], 8, 0 }, + { "do", &njs_lexer_kws[14], 2, 0 }, + { "enum", &njs_lexer_kws[37], 4, 0 }, + { "extends", &njs_lexer_kws[38], 7, 0 }, + { "instanceof", &njs_lexer_kws[5], 10, 0 }, + { "public", &njs_lexer_kws[45], 6, 0 }, + { "static", &njs_lexer_kws[46], 6, 0 }, + { "in", &njs_lexer_kws[3], 2, 0 }, + { "await", &njs_lexer_kws[33], 5, 0 }, + { "private", &njs_lexer_kws[43], 7, 0 }, + { NULL, NULL, 0, 0 }, + { "debugger", &njs_lexer_kws[36], 8, 0 }, + { "for", &njs_lexer_kws[15], 3, 1 }, + { NULL, NULL, 0, 0 }, + { "catch", &njs_lexer_kws[25], 5, 0 }, + { NULL, NULL, 0, 0 }, + { "super", &njs_lexer_kws[47], 5, 2 }, + { NULL, NULL, 0, 0 }, + { "const", &njs_lexer_kws[35], 5, 0 }, + { NULL, NULL, 0, 0 }, + { "false", &njs_lexer_kws[1], 5, 0 }, + { "with", &njs_lexer_kws[23], 4, 0 }, + { "implements", &njs_lexer_kws[39], 10, 0 }, + { "this", &njs_lexer_kws[30], 4, 0 }, + { "let", &njs_lexer_kws[41], 3, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { "true", &njs_lexer_kws[2], 4, 0 }, + { NULL, NULL, 0, 0 }, + { "export", &njs_lexer_kws[29], 6, 0 }, + { NULL, NULL, 0, 0 }, + { "interface", &njs_lexer_kws[40], 9, 0 }, + { NULL, NULL, 0, 0 }, + { "eval", &njs_lexer_kws[32], 4, 0 }, + { "protected", &njs_lexer_kws[44], 9, 0 }, + { "while", &njs_lexer_kws[13], 5, 0 }, + { NULL, NULL, 0, 0 }, + { "void", &njs_lexer_kws[6], 4, 0 }, + { NULL, NULL, 0, 0 }, + { "return", &njs_lexer_kws[22], 6, 0 }, + { NULL, NULL, 0, 0 }, + { "delete", &njs_lexer_kws[8], 6, 0 }, + { "yield", &njs_lexer_kws[9], 5, 0 }, + { "null", &njs_lexer_kws[0], 4, 0 }, + { "throw", &njs_lexer_kws[27], 5, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { "import", &njs_lexer_kws[28], 6, 0 }, + { NULL, NULL, 0, 0 }, + { "switch", &njs_lexer_kws[18], 6, 0 }, + { "try", &njs_lexer_kws[24], 3, 0 }, + { "function", &njs_lexer_kws[21], 8, 0 }, + { NULL, NULL, 0, 0 }, + { "if", &njs_lexer_kws[11], 2, 0 }, + { "break", &njs_lexer_kws[16], 5, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { NULL, NULL, 0, 0 }, + { "var", &njs_lexer_kws[10], 3, 4 }, + { NULL, NULL, 0, 0 }, + { "default", &njs_lexer_kws[20], 7, 0 }, + { "arguments", &njs_lexer_kws[31], 9, 6 }, + { "finally", &njs_lexer_kws[26], 7, 0 }, + { NULL, NULL, 0, 0 }, + { "else", &njs_lexer_kws[12], 4, 0 }, + { "class", &njs_lexer_kws[34], 5, 7 }, + { "new", &njs_lexer_kws[7], 3, 8 }, + { NULL, NULL, 0, 0 }, + { "package", &njs_lexer_kws[42], 7, 11 }, + { "typeof", &njs_lexer_kws[4], 6, 0 }, + { NULL, NULL, 0, 0 }, +}; + + +#endif /* _NJS_LEXER_TABLES_H_INCLUDED_ */ diff --git a/src/njs_module.c b/src/njs_module.c index b8224396..1ae0c889 100644 --- a/src/njs_module.c +++ b/src/njs_module.c @@ -103,8 +103,8 @@ njs_parser_module(njs_vm_t *vm, njs_parser_t *parser) njs_int_t ret; njs_str_t name, text; njs_lexer_t *prev, lexer; - njs_token_t token; njs_module_t *module; + njs_token_t token; njs_parser_node_t *node; njs_module_info_t info; diff --git a/src/njs_parser.c b/src/njs_parser.c index 992ea7a8..b9dfdf72 100644 --- a/src/njs_parser.c +++ b/src/njs_parser.c @@ -191,9 +191,9 @@ njs_parser_scope_begin(njs_vm_t *vm, njs_parser_t *parser, njs_scope_t type) scope->argument_closures = 0; njs_queue_init(&scope->nested); - njs_lvlhsh_init(&scope->labels); - njs_lvlhsh_init(&scope->variables); - njs_lvlhsh_init(&scope->references); + njs_rbtree_init(&scope->variables, njs_parser_scope_rbtree_compare); + njs_rbtree_init(&scope->labels, njs_parser_scope_rbtree_compare); + njs_rbtree_init(&scope->references, njs_parser_scope_rbtree_compare); values = NULL; @@ -243,6 +243,27 @@ njs_parser_scope_end(njs_vm_t *vm, njs_parser_t *parser) } +intptr_t +njs_parser_scope_rbtree_compare(njs_rbtree_node_t *node1, + njs_rbtree_node_t *node2) +{ + njs_variable_node_t *lex_node1, *lex_node2; + + lex_node1 = (njs_variable_node_t *) node1; + lex_node2 = (njs_variable_node_t *) node2; + + if (lex_node1->key < lex_node2->key) { + return -1; + } + + if (lex_node1->key > lex_node2->key) { + return 1; + } + + return 0; +} + + static njs_token_t njs_parser_statement_chain(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token, njs_bool_t top) @@ -479,14 +500,14 @@ njs_parser_block(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token) static njs_parser_node_t * -njs_parser_variable_node(njs_vm_t *vm, njs_parser_t *parser, njs_str_t *name, - uint32_t hash, njs_variable_type_t type) +njs_parser_variable_node(njs_vm_t *vm, njs_parser_t *parser, + uintptr_t unique_id, njs_variable_type_t type) { njs_int_t ret; njs_variable_t *var; njs_parser_node_t *node; - var = njs_variable_add(vm, parser->scope, name, hash, type); + var = njs_variable_add(vm, parser->scope, unique_id, type); if (njs_slow_path(var == NULL)) { return NULL; } @@ -509,7 +530,7 @@ njs_parser_variable_node(njs_vm_t *vm, njs_parser_t *parser, njs_str_t *name, return NULL; } - ret = njs_variable_reference(vm, parser->scope, node, name, hash, + ret = njs_variable_reference(vm, parser->scope, node, unique_id, NJS_DECLARATION); if (njs_slow_path(ret != NJS_OK)) { return NULL; @@ -522,23 +543,23 @@ njs_parser_variable_node(njs_vm_t *vm, njs_parser_t *parser, njs_str_t *name, static njs_token_t njs_parser_labelled_statement(njs_vm_t *vm, njs_parser_t *parser) { - uint32_t hash; + uintptr_t unique_id; njs_int_t ret; njs_str_t name; njs_token_t token; njs_variable_t *label; name = *njs_parser_text(parser); - hash = njs_parser_key_hash(parser); + unique_id = njs_parser_key_hash(parser); - label = njs_label_find(vm, parser->scope, &name, hash); + label = njs_label_find(vm, parser->scope, unique_id); if (njs_slow_path(label != NULL)) { njs_parser_syntax_error(vm, parser, "Label \"%V\" " "has already been declared", &name); return NJS_TOKEN_ILLEGAL; } - label = njs_label_add(vm, parser->scope, &name, hash); + label = njs_label_add(vm, parser->scope, unique_id); if (njs_slow_path(label == NULL)) { return NJS_TOKEN_ERROR; } @@ -565,7 +586,7 @@ njs_parser_labelled_statement(njs_vm_t *vm, njs_parser_t *parser) return NJS_TOKEN_ERROR; } - ret = njs_label_remove(vm, parser->scope, &name, hash); + ret = njs_label_remove(vm, parser->scope, unique_id); if (njs_slow_path(ret != NJS_OK)) { return NJS_TOKEN_ERROR; } @@ -831,7 +852,6 @@ static njs_token_t njs_parser_lambda_argument(njs_vm_t *vm, njs_parser_t *parser, njs_index_t index) { - njs_int_t ret; njs_variable_t *arg; arg = njs_parser_variable_add(vm, parser, NJS_VARIABLE_VAR); @@ -846,11 +866,7 @@ njs_parser_lambda_argument(njs_vm_t *vm, njs_parser_t *parser, } arg->index = index; - - ret = njs_name_copy(vm, &arg->name, njs_parser_text(parser)); - if (njs_slow_path(ret != NJS_OK)) { - return NJS_TOKEN_ERROR; - } + arg->unique_id = njs_parser_key_hash(parser); return njs_parser_token(vm, parser); } @@ -1015,8 +1031,8 @@ njs_parser_return_statement(njs_vm_t *vm, njs_parser_t *parser) static njs_token_t -njs_parser_var_statement(njs_vm_t *vm, njs_parser_t *parser, njs_token_t parent, - njs_bool_t var_in) +njs_parser_var_statement(njs_vm_t *vm, njs_parser_t *parser, + njs_token_t parent, njs_bool_t var_in) { njs_token_t token; njs_parser_node_t *left, *stmt, *name, *assign, *expr; @@ -1043,9 +1059,7 @@ njs_parser_var_statement(njs_vm_t *vm, njs_parser_t *parser, njs_token_t parent, return NJS_TOKEN_ILLEGAL; } - name = njs_parser_variable_node(vm, parser, - njs_parser_text(parser), - njs_parser_key_hash(parser), + name = njs_parser_variable_node(vm, parser, njs_parser_key_hash(parser), type); if (njs_slow_path(name == NULL)) { return NJS_TOKEN_ERROR; @@ -1567,7 +1581,7 @@ static njs_token_t njs_parser_brk_statement(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token) { - uint32_t hash; + uintptr_t unique_id; njs_int_t ret; njs_str_t name; njs_parser_node_t *node; @@ -1589,9 +1603,9 @@ njs_parser_brk_statement(njs_vm_t *vm, njs_parser_t *parser, case NJS_TOKEN_NAME: name = *njs_parser_text(parser); - hash = njs_parser_key_hash(parser); + unique_id = njs_parser_key_hash(parser); - if (njs_label_find(vm, parser->scope, &name, hash) == NULL) { + if (njs_label_find(vm, parser->scope, unique_id) == NULL) { njs_parser_syntax_error(vm, parser, "Undefined label \"%V\"", &name); return NJS_TOKEN_ILLEGAL; @@ -1666,8 +1680,7 @@ njs_parser_try_statement(njs_vm_t *vm, njs_parser_t *parser) return NJS_TOKEN_ERROR; } - node = njs_parser_variable_node(vm, parser, njs_parser_text(parser), - njs_parser_key_hash(parser), + node = njs_parser_variable_node(vm, parser, njs_parser_key_hash(parser), NJS_VARIABLE_CATCH); if (njs_slow_path(node == NULL)) { return NJS_TOKEN_ERROR; @@ -1818,8 +1831,7 @@ njs_parser_import_statement(njs_vm_t *vm, njs_parser_t *parser) return NJS_TOKEN_ILLEGAL; } - name = njs_parser_variable_node(vm, parser, njs_parser_text(parser), - njs_parser_key_hash(parser), + name = njs_parser_variable_node(vm, parser, njs_parser_key_hash(parser), NJS_VARIABLE_VAR); if (njs_slow_path(name == NULL)) { return NJS_TOKEN_ERROR; diff --git a/src/njs_parser.h b/src/njs_parser.h index 1404f7e9..c04c5029 100644 --- a/src/njs_parser.h +++ b/src/njs_parser.h @@ -15,9 +15,9 @@ struct njs_parser_scope_s { njs_queue_t nested; njs_parser_scope_t *parent; - njs_lvlhsh_t labels; - njs_lvlhsh_t variables; - njs_lvlhsh_t references; + njs_rbtree_t variables; + njs_rbtree_t labels; + njs_rbtree_t references; #define NJS_SCOPE_INDEX_LOCAL 0 #define NJS_SCOPE_INDEX_CLOSURE 1 @@ -77,12 +77,21 @@ struct njs_parser_s { }; +typedef struct { + NJS_RBTREE_NODE (node); + uintptr_t key; + njs_parser_node_t *parser_node; +} njs_parser_rbtree_node_t; + + +intptr_t njs_parser_scope_rbtree_compare(njs_rbtree_node_t *node1, + njs_rbtree_node_t *node2); njs_int_t njs_parser(njs_vm_t *vm, njs_parser_t *parser, njs_parser_t *prev); njs_token_t njs_parser_expression(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token); -njs_token_t njs_parser_assignment_expression(njs_vm_t *vm, - njs_parser_t *parser, njs_token_t token); +njs_token_t njs_parser_assignment_expression(njs_vm_t *vm, njs_parser_t *parser, + njs_token_t token); njs_token_t njs_parser_function_expression(njs_vm_t *vm, njs_parser_t *parser); njs_int_t njs_parser_match_arrow_expression(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token); @@ -139,19 +148,19 @@ void njs_parser_node_error(njs_vm_t *vm, njs_parser_node_t *node, #define njs_parser_text(parser) \ - &(parser)->lexer->lexer_token->text + &(parser)->lexer->token->text #define njs_parser_key_hash(parser) \ - (parser)->lexer->lexer_token->key_hash + (parser)->lexer->token->unique_id #define njs_parser_number(parser) \ - (parser)->lexer->lexer_token->number + (parser)->lexer->token->number #define njs_parser_token_line(parser) \ - (parser)->lexer->lexer_token->token_line + (parser)->lexer->token->line #define njs_parser_syntax_error(vm, parser, fmt, ...) \ @@ -263,7 +272,7 @@ njs_inline njs_variable_t * njs_parser_variable_add(njs_vm_t *vm, njs_parser_t *parser, njs_variable_type_t type) { - return njs_variable_add(vm, parser->scope, njs_parser_text(parser), + return njs_variable_add(vm, parser->scope, njs_parser_key_hash(parser), type); } @@ -273,7 +282,6 @@ njs_parser_variable_reference(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *node, njs_reference_type_t type) { return njs_variable_reference(vm, parser->scope, node, - njs_parser_text(parser), njs_parser_key_hash(parser), type); } @@ -310,7 +318,4 @@ njs_function_scope(njs_parser_scope_t *scope, njs_bool_t any) } -extern const njs_lvlhsh_proto_t njs_keyword_hash_proto; - - #endif /* _NJS_PARSER_H_INCLUDED_ */ diff --git a/src/njs_parser_terminal.c b/src/njs_parser_terminal.c index e1aeecbb..c079f720 100644 --- a/src/njs_parser_terminal.c +++ b/src/njs_parser_terminal.c @@ -9,8 +9,8 @@ static njs_parser_node_t *njs_parser_reference(njs_vm_t *vm, - njs_parser_t *parser, njs_token_t token, njs_str_t *name, uint32_t hash, - uint32_t token_line); + njs_parser_t *parser, njs_token_t token, njs_str_t *name, + uintptr_t hash, uint32_t token_line); static njs_token_t njs_parser_object(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *obj); static njs_int_t njs_parser_object_property(njs_vm_t *vm, njs_parser_t *parser, @@ -161,16 +161,16 @@ njs_parser_terminal(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token) break; - case NJS_TOKEN_BOOLEAN: - num = njs_parser_number(parser); + case NJS_TOKEN_TRUE: + case NJS_TOKEN_FALSE: njs_thread_log_debug("JS: boolean: %V", njs_parser_text(parser)); - node = njs_parser_node_new(vm, parser, NJS_TOKEN_BOOLEAN); + node = njs_parser_node_new(vm, parser, parser->lexer->token->type); if (njs_slow_path(node == NULL)) { return NJS_TOKEN_ERROR; } - if (num == 0) { + if (parser->lexer->token->type == NJS_TOKEN_FALSE) { node->u.value = njs_value_false; } else { @@ -180,11 +180,9 @@ njs_parser_terminal(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token) break; default: - node = njs_parser_reference(vm, parser, token, - njs_parser_text(parser), + node = njs_parser_reference(vm, parser, token, njs_parser_text(parser), njs_parser_key_hash(parser), njs_parser_token_line(parser)); - if (njs_slow_path(node == NULL)) { return NJS_TOKEN_ERROR; } @@ -200,7 +198,7 @@ njs_parser_terminal(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token) static njs_parser_node_t * njs_parser_reference(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token, - njs_str_t *name, uint32_t hash, uint32_t token_line) + njs_str_t *name, uintptr_t unique_id, uint32_t token_line) { njs_int_t ret; njs_variable_t *var; @@ -234,13 +232,13 @@ njs_parser_reference(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token, node->token_line = token_line; - ret = njs_variable_reference(vm, scope, node, name, hash, + ret = njs_variable_reference(vm, scope, node, unique_id, NJS_REFERENCE); if (njs_slow_path(ret != NJS_OK)) { return NULL; } - var = njs_variable_add(vm, scope, name, hash, NJS_VARIABLE_VAR); + var = njs_variable_add(vm, scope, unique_id, NJS_VARIABLE_VAR); if (njs_slow_path(var == NULL)) { return NULL; } @@ -269,13 +267,12 @@ njs_parser_reference(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token, node->token_line = token_line; - ret = njs_variable_reference(vm, scope, node, name, hash, - NJS_REFERENCE); + ret = njs_variable_reference(vm, scope, node, unique_id, NJS_REFERENCE); if (njs_slow_path(ret != NJS_OK)) { return NULL; } - var = njs_variable_add(vm, scope, name, hash, NJS_VARIABLE_VAR); + var = njs_variable_add(vm, scope, unique_id, NJS_VARIABLE_VAR); if (njs_slow_path(var == NULL)) { return NULL; } @@ -290,7 +287,7 @@ njs_parser_reference(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token, node->token_line = token_line; - ret = njs_variable_reference(vm, parser->scope, node, name, hash, + ret = njs_variable_reference(vm, parser->scope, node, unique_id, NJS_REFERENCE); if (njs_slow_path(ret != NJS_OK)) { return NULL; @@ -310,7 +307,8 @@ njs_parser_reference(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token, static njs_token_t njs_parser_object(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *obj) { - uint32_t hash, token_line; + uintptr_t unique_id; + uint32_t token_line; njs_int_t ret, __proto__; njs_str_t name; njs_bool_t computed, proto_init; @@ -324,7 +322,7 @@ njs_parser_object(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *obj) lexer = parser->lexer; /* GCC and Clang complain about uninitialized values. */ - hash = 0; + unique_id = 0; token_line = 0; __proto__ = 0; property = NULL; @@ -349,7 +347,7 @@ njs_parser_object(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *obj) if (token == NJS_TOKEN_NAME || lexer->keyword) { name = *njs_parser_text(parser); - hash = njs_parser_key_hash(parser); + unique_id = njs_parser_key_hash(parser); token_line = njs_parser_token_line(parser); property = njs_parser_node_string(vm, parser); @@ -494,7 +492,7 @@ njs_parser_object(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *obj) } expression = njs_parser_reference(vm, parser, lexer->prev_token, - &name, hash, token_line); + &name, unique_id, token_line); if (njs_slow_path(expression == NULL)) { return NJS_TOKEN_ERROR; } @@ -867,7 +865,7 @@ njs_parser_template_string(njs_vm_t *vm, njs_parser_t *parser) njs_parser_node_t *node; lexer = parser->lexer; - text = &lexer->lexer_token->text; + text = &lexer->token->text; text->start = lexer->start; diff --git a/src/njs_shell.c b/src/njs_shell.c index cad7e3ab..7736b472 100644 --- a/src/njs_shell.c +++ b/src/njs_shell.c @@ -47,7 +47,7 @@ typedef struct { size_t length; njs_arr_t *completions; njs_arr_t *suffix_completions; - njs_lvlhsh_each_t lhe; + njs_rbtree_node_t *node; enum { NJS_COMPLETION_VAR = 0, @@ -929,13 +929,15 @@ njs_editline_init(void) static char * njs_completion_generator(const char *text, int state) { - char *completion; - size_t len; - njs_str_t expression, *suffix; - const char *p; - njs_vm_t *vm; - njs_variable_t *var; - njs_completion_t *cmpl; + char *completion; + size_t len; + njs_str_t expression, *suffix; + njs_vm_t *vm; + const char *p; + njs_rbtree_t *variables; + njs_completion_t *cmpl; + njs_variable_node_t *var_node; + const njs_lexer_entry_t *lex_entry; vm = njs_console.vm; cmpl = &njs_console.completion; @@ -946,7 +948,9 @@ njs_completion_generator(const char *text, int state) cmpl->length = njs_strlen(text); cmpl->suffix_completions = NULL; - njs_lvlhsh_each_init(&cmpl->lhe, &njs_variables_hash_proto); + if (vm->parser != NULL) { + cmpl->node = njs_rbtree_min(&vm->parser->scope->variables); + } } next: @@ -957,21 +961,24 @@ next: njs_next_phase(cmpl); } - for ( ;; ) { - var = njs_lvlhsh_each(&vm->parser->scope->variables, - &cmpl->lhe); + variables = &vm->parser->scope->variables; - if (var == NULL) { + while (njs_rbtree_is_there_successor(variables, cmpl->node)) { + var_node = (njs_variable_node_t *) cmpl->node; + + lex_entry = njs_lexer_entry(var_node->key); + if (lex_entry == NULL) { break; } - if (var->name.length < cmpl->length) { - continue; - } + cmpl->node = njs_rbtree_node_successor(variables, cmpl->node); - if (njs_strncmp(text, var->name.start, cmpl->length) == 0) { - return njs_editline(&var->name); + if (lex_entry->name.length >= cmpl->length + && njs_strncmp(text, lex_entry->name.start, cmpl->length) == 0) + { + return njs_editline(&lex_entry->name); } + } njs_next_phase(cmpl); diff --git a/src/njs_variable.c b/src/njs_variable.c index 63f9e471..b42b26fa 100644 --- a/src/njs_variable.c +++ b/src/njs_variable.c @@ -10,51 +10,20 @@ static njs_variable_t *njs_variable_scope_add(njs_vm_t *vm, - njs_parser_scope_t *scope, njs_lvlhsh_query_t *lhq, - njs_variable_type_t type); + njs_parser_scope_t *scope, uintptr_t unique_id, njs_variable_type_t type); static njs_int_t njs_variable_reference_resolve(njs_vm_t *vm, njs_variable_reference_t *vr, njs_parser_scope_t *node_scope); -static njs_variable_t *njs_variable_alloc(njs_vm_t *vm, njs_str_t *name, +static njs_variable_t *njs_variable_alloc(njs_vm_t *vm, uintptr_t unique_id, njs_variable_type_t type); -static njs_int_t -njs_variables_hash_test(njs_lvlhsh_query_t *lhq, void *data) -{ - njs_variable_t *var; - - var = data; - - if (njs_strstr_eq(&lhq->key, &var->name)) { - return NJS_OK; - } - - return NJS_DECLINED; -} - - -const njs_lvlhsh_proto_t njs_variables_hash_proto - njs_aligned(64) = -{ - NJS_LVLHSH_DEFAULT, - njs_variables_hash_test, - njs_lvlhsh_alloc, - njs_lvlhsh_free, -}; - - njs_variable_t * -njs_variable_add(njs_vm_t *vm, njs_parser_scope_t *scope, njs_str_t *name, - uint32_t hash, njs_variable_type_t type) +njs_variable_add(njs_vm_t *vm, njs_parser_scope_t *scope, uintptr_t unique_id, + njs_variable_type_t type) { - njs_variable_t *var; - njs_lvlhsh_query_t lhq; - - lhq.key_hash = hash; - lhq.key = *name; - lhq.proto = &njs_variables_hash_proto; + njs_variable_t *var; - var = njs_variable_scope_add(vm, scope, &lhq, type); + var = njs_variable_scope_add(vm, scope, unique_id, type); if (njs_slow_path(var == NULL)) { return NULL; } @@ -64,7 +33,7 @@ njs_variable_add(njs_vm_t *vm, njs_parser_scope_t *scope, njs_str_t *name, do { scope = scope->parent; - var = njs_variable_scope_add(vm, scope, &lhq, type); + var = njs_variable_scope_add(vm, scope, unique_id, type); if (njs_slow_path(var == NULL)) { return NULL; } @@ -81,35 +50,27 @@ njs_variable_add(njs_vm_t *vm, njs_parser_scope_t *scope, njs_str_t *name, njs_int_t -njs_variables_copy(njs_vm_t *vm, njs_lvlhsh_t *variables, - njs_lvlhsh_t *prev_variables) +njs_variables_copy(njs_vm_t *vm, njs_rbtree_t *variables, + njs_rbtree_t *prev_variables) { - njs_int_t ret; - njs_variable_t *var; - njs_lvlhsh_each_t lhe; - njs_lvlhsh_query_t lhq; - - njs_lvlhsh_each_init(&lhe, &njs_variables_hash_proto); + njs_rbtree_node_t *node; + njs_variable_node_t *var_node; - lhq.proto = &njs_variables_hash_proto; - lhq.replace = 0; - lhq.pool = vm->mem_pool; + node = njs_rbtree_min(prev_variables); - for ( ;; ) { - var = njs_lvlhsh_each(prev_variables, &lhe); + while (njs_rbtree_is_there_successor(prev_variables, node)) { + var_node = (njs_variable_node_t *) node; - if (var == NULL) { - break; + var_node = njs_variable_node_alloc(vm, var_node->variable, + var_node->key); + if (njs_slow_path(var_node == NULL)) { + njs_memory_error(vm); + return NJS_ERROR; } - lhq.value = var; - lhq.key = var->name; - lhq.key_hash = njs_djb_hash(var->name.start, var->name.length); + njs_rbtree_insert(variables, &var_node->node); - ret = njs_lvlhsh_insert(variables, &lhq); - if (njs_slow_path(ret != NJS_OK)) { - return NJS_ERROR; - } + node = njs_rbtree_node_successor(prev_variables, node); } return NJS_OK; @@ -118,13 +79,19 @@ njs_variables_copy(njs_vm_t *vm, njs_lvlhsh_t *variables, static njs_variable_t * njs_variable_scope_add(njs_vm_t *vm, njs_parser_scope_t *scope, - njs_lvlhsh_query_t *lhq, njs_variable_type_t type) + uintptr_t unique_id, njs_variable_type_t type) { - njs_int_t ret; - njs_variable_t *var; + njs_variable_t *var; + njs_rbtree_node_t *node; + njs_variable_node_t var_node, *var_node_new; + const njs_lexer_entry_t *entry; - if (njs_lvlhsh_find(&scope->variables, lhq) == NJS_OK) { - var = lhq->value; + var_node.key = unique_id; + + node = njs_rbtree_find(&scope->variables, &var_node.node); + + if (node != NULL) { + var = ((njs_variable_node_t *) node)->variable; if (scope->module || scope->type == NJS_SCOPE_BLOCK) { @@ -149,162 +116,118 @@ njs_variable_scope_add(njs_vm_t *vm, njs_parser_scope_t *scope, return var; } - var = njs_variable_alloc(vm, &lhq->key, type); + var = njs_variable_alloc(vm, unique_id, type); if (njs_slow_path(var == NULL)) { - return NULL; + goto memory_error; } - lhq->replace = 0; - lhq->value = var; - lhq->pool = vm->mem_pool; + var_node_new = njs_variable_node_alloc(vm, var, unique_id); + if (njs_slow_path(var_node_new == NULL)) { + goto memory_error; + } - ret = njs_lvlhsh_insert(&scope->variables, lhq); + njs_rbtree_insert(&scope->variables, &var_node_new->node); - if (njs_fast_path(ret == NJS_OK)) { - return var; - } + return var; - njs_mp_free(vm->mem_pool, var->name.start); - njs_mp_free(vm->mem_pool, var); +memory_error: - njs_type_error(vm, "lvlhsh insert failed"); + njs_memory_error(vm); return NULL; fail: + entry = njs_lexer_entry(unique_id); + njs_parser_syntax_error(vm, vm->parser, "\"%V\" has already been declared", - &lhq->key); + &entry->name); return NULL; } njs_variable_t * -njs_label_add(njs_vm_t *vm, njs_parser_scope_t *scope, njs_str_t *name, - uint32_t hash) +njs_label_add(njs_vm_t *vm, njs_parser_scope_t *scope, uintptr_t unique_id) { - njs_int_t ret; - njs_variable_t *label; - njs_lvlhsh_query_t lhq; + njs_variable_t *label; + njs_rbtree_node_t *node; + njs_variable_node_t var_node, *var_node_new; - lhq.key_hash = hash; - lhq.key = *name; - lhq.proto = &njs_variables_hash_proto; + var_node.key = unique_id; + + node = njs_rbtree_find(&scope->labels, &var_node.node); - if (njs_lvlhsh_find(&scope->labels, &lhq) == NJS_OK) { - return lhq.value; + if (node != NULL) { + return ((njs_variable_node_t *) node)->variable; } - label = njs_variable_alloc(vm, &lhq.key, NJS_VARIABLE_CONST); + label = njs_variable_alloc(vm, unique_id, NJS_VARIABLE_CONST); if (njs_slow_path(label == NULL)) { - return label; + goto memory_error; } - lhq.replace = 0; - lhq.value = label; - lhq.pool = vm->mem_pool; + var_node_new = njs_variable_node_alloc(vm, label, unique_id); + if (njs_slow_path(var_node_new == NULL)) { + goto memory_error; + } - ret = njs_lvlhsh_insert(&scope->labels, &lhq); + njs_rbtree_insert(&scope->labels, &var_node_new->node); - if (njs_fast_path(ret == NJS_OK)) { - return label; - } + return label; - njs_mp_free(vm->mem_pool, label->name.start); - njs_mp_free(vm->mem_pool, label); +memory_error: - njs_internal_error(vm, "lvlhsh insert failed"); + njs_memory_error(vm); return NULL; } njs_int_t -njs_label_remove(njs_vm_t *vm, njs_parser_scope_t *scope, njs_str_t *name, - uint32_t hash) +njs_label_remove(njs_vm_t *vm, njs_parser_scope_t *scope, uintptr_t unique_id) { - njs_int_t ret; - njs_variable_t *label; - njs_lvlhsh_query_t lhq; - - lhq.key_hash = hash; - lhq.key = *name; - lhq.proto = &njs_variables_hash_proto; - lhq.pool = vm->mem_pool; + njs_rbtree_node_t *node; + njs_variable_node_t var_node; - ret = njs_lvlhsh_delete(&scope->labels, &lhq); + var_node.key = unique_id; - if (njs_fast_path(ret == NJS_OK)) { - label = lhq.value; - njs_mp_free(vm->mem_pool, label->name.start); - njs_mp_free(vm->mem_pool, label); - - } else { - njs_internal_error(vm, "lvlhsh delete failed"); + node = njs_rbtree_find(&scope->labels, &var_node.node); + if (njs_slow_path(node == NULL)) { + njs_internal_error(vm, "failed to find label while removing"); + return NJS_ERROR; } - return ret; -} - + njs_rbtree_delete(&scope->labels, (njs_rbtree_part_t *) node); + njs_variable_node_free(vm, (njs_variable_node_t *) node); -static njs_int_t -njs_reference_hash_test(njs_lvlhsh_query_t *lhq, void *data) -{ - njs_parser_node_t *node; - - node = data; - - if (njs_strstr_eq(&lhq->key, &node->u.reference.name)) { - return NJS_OK; - } - - return NJS_DECLINED; + return NJS_OK; } -const njs_lvlhsh_proto_t njs_references_hash_proto - njs_aligned(64) = -{ - NJS_LVLHSH_DEFAULT, - njs_reference_hash_test, - njs_lvlhsh_alloc, - njs_lvlhsh_free, -}; - - njs_int_t njs_variable_reference(njs_vm_t *vm, njs_parser_scope_t *scope, - njs_parser_node_t *node, njs_str_t *name, uint32_t hash, - njs_reference_type_t type) + njs_parser_node_t *node, uintptr_t unique_id, njs_reference_type_t type) { - njs_int_t ret; - njs_lvlhsh_query_t lhq; njs_variable_reference_t *vr; + njs_parser_rbtree_node_t *rb_node; vr = &node->u.reference; - ret = njs_name_copy(vm, &vr->name, name); + vr->unique_id = unique_id; + vr->type = type; - if (njs_fast_path(ret == NJS_OK)) { - vr->hash = hash; - vr->type = type; - - lhq.key_hash = hash; - lhq.key = vr->name; - lhq.proto = &njs_references_hash_proto; - lhq.replace = 0; - lhq.value = node; - lhq.pool = vm->mem_pool; + rb_node = njs_mp_alloc(vm->mem_pool, sizeof(njs_parser_rbtree_node_t)); + if (njs_slow_path(rb_node == NULL)) { + return NJS_ERROR; + } - ret = njs_lvlhsh_insert(&scope->references, &lhq); + rb_node->key = unique_id; + rb_node->parser_node = node; - if (njs_fast_path(ret != NJS_ERROR)) { - ret = NJS_OK; - } - } + njs_rbtree_insert(&scope->references, &rb_node->node); - return ret; + return NJS_OK; } @@ -315,8 +238,9 @@ njs_variables_scope_resolve(njs_vm_t *vm, njs_parser_scope_t *scope, njs_int_t ret; njs_queue_t *nested; njs_queue_link_t *lnk; + njs_rbtree_node_t *rb_node; njs_parser_node_t *node; - njs_lvlhsh_each_t lhe; + njs_parser_rbtree_node_t *parser_rb_node; njs_variable_reference_t *vr; nested = &scope->nested; @@ -332,10 +256,11 @@ njs_variables_scope_resolve(njs_vm_t *vm, njs_parser_scope_t *scope, return NJS_ERROR; } - njs_lvlhsh_each_init(&lhe, &njs_variables_hash_proto); + rb_node = njs_rbtree_min(&scope->references); - for ( ;; ) { - node = njs_lvlhsh_each(&scope->references, &lhe); + while (njs_rbtree_is_there_successor(&scope->references, rb_node)) { + parser_rb_node = (njs_parser_rbtree_node_t *) rb_node; + node = parser_rb_node->parser_node; if (node == NULL) { break; @@ -346,15 +271,19 @@ njs_variables_scope_resolve(njs_vm_t *vm, njs_parser_scope_t *scope, if (closure) { ret = njs_variable_reference_resolve(vm, vr, node->scope); if (njs_slow_path(ret != NJS_OK)) { - continue; + goto next; } if (vr->scope_index == NJS_SCOPE_INDEX_LOCAL) { - continue; + goto next; } } (void) njs_variable_resolve(vm, node); + + next: + + rb_node = njs_rbtree_node_successor(&scope->references, rb_node); } } @@ -487,26 +416,25 @@ njs_variable_resolve(njs_vm_t *vm, njs_parser_node_t *node) njs_variable_t * -njs_label_find(njs_vm_t *vm, njs_parser_scope_t *scope, njs_str_t *name, - uint32_t hash) +njs_label_find(njs_vm_t *vm, njs_parser_scope_t *scope, uintptr_t unique_id) { - njs_lvlhsh_query_t lhq; + njs_rbtree_node_t *node; + njs_variable_node_t var_node; - lhq.key_hash = hash; - lhq.key = *name; - lhq.proto = &njs_variables_hash_proto; + var_node.key = unique_id; - for ( ;; ) { - if (njs_lvlhsh_find(&scope->labels, &lhq) == NJS_OK) { - return lhq.value; + do { + node = njs_rbtree_find(&scope->labels, &var_node.node); + + if (node != NULL) { + return ((njs_variable_node_t *) node)->variable; } scope = scope->parent; - if (scope == NULL) { - return NULL; - } - } + } while (scope != NULL); + + return NULL; } @@ -514,19 +442,20 @@ static njs_int_t njs_variable_reference_resolve(njs_vm_t *vm, njs_variable_reference_t *vr, njs_parser_scope_t *node_scope) { - njs_lvlhsh_query_t lhq; - njs_parser_scope_t *scope, *previous; + njs_rbtree_node_t *node; + njs_parser_scope_t *scope, *previous; + njs_variable_node_t var_node; - lhq.key_hash = vr->hash; - lhq.key = vr->name; - lhq.proto = &njs_variables_hash_proto; + var_node.key = vr->unique_id; scope = node_scope; previous = NULL; for ( ;; ) { - if (njs_lvlhsh_find(&scope->variables, &lhq) == NJS_OK) { - vr->variable = lhq.value; + node = njs_rbtree_find(&scope->variables, &var_node.node); + + if (node != NULL) { + vr->variable = ((njs_variable_node_t *) node)->variable; if (scope->type == NJS_SCOPE_BLOCK && vr->variable->type == NJS_VARIABLE_VAR) @@ -624,9 +553,8 @@ njs_scope_next_index(njs_vm_t *vm, njs_parser_scope_t *scope, static njs_variable_t * -njs_variable_alloc(njs_vm_t *vm, njs_str_t *name, njs_variable_type_t type) +njs_variable_alloc(njs_vm_t *vm, uintptr_t unique_id, njs_variable_type_t type) { - njs_int_t ret; njs_variable_t *var; var = njs_mp_zalloc(vm->mem_pool, sizeof(njs_variable_t)); @@ -635,24 +563,15 @@ njs_variable_alloc(njs_vm_t *vm, njs_str_t *name, njs_variable_type_t type) return NULL; } + var->unique_id = unique_id; var->type = type; - ret = njs_name_copy(vm, &var->name, name); - - if (njs_fast_path(ret == NJS_OK)) { - return var; - } - - njs_mp_free(vm->mem_pool, var); - - njs_memory_error(vm); - - return NULL; + return var; } njs_int_t -njs_name_copy(njs_vm_t *vm, njs_str_t *dst, njs_str_t *src) +njs_name_copy(njs_vm_t *vm, njs_str_t *dst, const njs_str_t *src) { dst->length = src->length; @@ -673,14 +592,29 @@ njs_name_copy(njs_vm_t *vm, njs_str_t *dst, njs_str_t *src) const njs_value_t * njs_vm_value(njs_vm_t *vm, const njs_str_t *name) { - njs_lvlhsh_query_t lhq; + njs_int_t ret; + njs_rbtree_node_t *rb_node; + njs_lvlhsh_query_t lhq; + njs_variable_node_t *node, var_node; - lhq.key_hash = njs_djb_hash(name->start, name->length); lhq.key = *name; - lhq.proto = &njs_variables_hash_proto; + lhq.key_hash = njs_djb_hash(name->start, name->length); + lhq.proto = &njs_lexer_hash_proto; + + ret = njs_lvlhsh_find(&vm->shared->keywords_hash, &lhq); + + if (njs_slow_path(ret != NJS_OK || lhq.value == NULL)) { + return &njs_value_undefined; + } + + var_node.key = (uintptr_t) lhq.value; + + rb_node = njs_rbtree_find(vm->variables_hash, &var_node.node); + + if (rb_node != NULL) { + node = (njs_variable_node_t *) rb_node; - if (njs_lvlhsh_find(&vm->variables_hash, &lhq) == NJS_OK) { - return njs_vmcode_operand(vm, ((njs_variable_t *) lhq.value)->index); + return njs_vmcode_operand(vm, node->variable->index); } return &njs_value_undefined; diff --git a/src/njs_variable.h b/src/njs_variable.h index 5d6c110b..f843a844 100644 --- a/src/njs_variable.h +++ b/src/njs_variable.h @@ -19,7 +19,7 @@ typedef enum { typedef struct { - njs_str_t name; + uintptr_t unique_id; njs_variable_type_t type:8; /* 3 bits */ uint8_t argument; @@ -40,8 +40,7 @@ typedef enum { typedef struct { njs_reference_type_t type; - uint32_t hash; - njs_str_t name; + uintptr_t unique_id; njs_variable_t *variable; njs_parser_scope_t *scope; njs_uint_t scope_index; /* NJS_SCOPE_INDEX_... */ @@ -49,26 +48,53 @@ typedef struct { } njs_variable_reference_t; +typedef struct { + NJS_RBTREE_NODE (node); + uintptr_t key; + njs_variable_t *variable; +} njs_variable_node_t; + + njs_variable_t *njs_variable_add(njs_vm_t *vm, njs_parser_scope_t *scope, - njs_str_t *name, uint32_t hash, njs_variable_type_t type); -njs_int_t njs_variables_copy(njs_vm_t *vm, njs_lvlhsh_t *variables, - njs_lvlhsh_t *prev_variables); + uintptr_t unique_id, njs_variable_type_t type); +njs_int_t njs_variables_copy(njs_vm_t *vm, njs_rbtree_t *variables, + njs_rbtree_t *prev_variables); njs_variable_t * njs_label_add(njs_vm_t *vm, njs_parser_scope_t *scope, - njs_str_t *name, uint32_t hash); + uintptr_t unique_id); njs_variable_t *njs_label_find(njs_vm_t *vm, njs_parser_scope_t *scope, - njs_str_t *name, uint32_t hash); + uintptr_t unique_id); njs_int_t njs_label_remove(njs_vm_t *vm, njs_parser_scope_t *scope, - njs_str_t *name, uint32_t hash); + uintptr_t unique_id); njs_int_t njs_variable_reference(njs_vm_t *vm, njs_parser_scope_t *scope, - njs_parser_node_t *node, njs_str_t *name, uint32_t hash, - njs_reference_type_t type); + njs_parser_node_t *node, uintptr_t unique_id, njs_reference_type_t type); njs_int_t njs_variables_scope_reference(njs_vm_t *vm, njs_parser_scope_t *scope); njs_index_t njs_scope_next_index(njs_vm_t *vm, njs_parser_scope_t *scope, njs_uint_t scope_index, const njs_value_t *default_value); -njs_int_t njs_name_copy(njs_vm_t *vm, njs_str_t *dst, njs_str_t *src); +njs_int_t njs_name_copy(njs_vm_t *vm, njs_str_t *dst, const njs_str_t *src); + + +njs_inline njs_variable_node_t * +njs_variable_node_alloc(njs_vm_t *vm, njs_variable_t *var, uintptr_t key) +{ + njs_variable_node_t *node; + + node = njs_mp_zalloc(vm->mem_pool, sizeof(njs_variable_node_t)); + + if (njs_fast_path(node != NULL)) { + node->key = key; + node->variable = var; + } + + return node; +} + -extern const njs_lvlhsh_proto_t njs_variables_hash_proto; +njs_inline void +njs_variable_node_free(njs_vm_t *vm, njs_variable_node_t *node) +{ + njs_mp_free(vm->mem_pool, node); +} #endif /* _NJS_VARIABLE_H_INCLUDED_ */ diff --git a/src/njs_vm.c b/src/njs_vm.c index ffb933ff..e6ec424c 100644 --- a/src/njs_vm.c +++ b/src/njs_vm.c @@ -188,7 +188,7 @@ njs_vm_compile(njs_vm_t *vm, u_char **start, u_char *end) vm->global_scope = generator.local_scope; vm->scope_size = generator.scope_size; - vm->variables_hash = scope->variables; + vm->variables_hash = &scope->variables; if (vm->options.init && !vm->options.accumulative) { ret = njs_vm_init(vm); diff --git a/src/njs_vm.h b/src/njs_vm.h index c24e5ac4..8e8b8d41 100644 --- a/src/njs_vm.h +++ b/src/njs_vm.h @@ -197,7 +197,7 @@ struct njs_vm_s { njs_lvlhsh_t external_prototypes_hash; - njs_lvlhsh_t variables_hash; + njs_rbtree_t *variables_hash; njs_lvlhsh_t values_hash; njs_arr_t *modules; diff --git a/utils/lexer_keyword.py b/utils/lexer_keyword.py new file mode 100755 index 00000000..535dd011 --- /dev/null +++ b/utils/lexer_keyword.py @@ -0,0 +1,245 @@ +import re, os + +global_keywords = [ + # Values. + + "null", + "false", + "true", + + # Operators. + + "in", + "typeof", + "instanceof", + "void", + "new", + "delete", + "yield", + + # Statements. + + "var", + "if", + "else", + "while", + "do", + "for", + "break", + "continue", + "switch", + "case", + "default", + "function", + "return", + "with", + "try", + "catch", + "finally", + "throw", + + # Module. + + "import", + "export", + + # Reserved words. + + "this", + "arguments", + "eval", + + "await", + "class", + "const", + "debugger", + "enum", + "extends", + "implements", + "interface", + "let", + "package", + "private", + "protected", + "public", + "static", + "super" +] + + +class Table: + def __init__(self, header): + self.buffer = [] + self.header = header + + def add(self, data): + self.buffer.append(data) + + def create(self): + result = [] + data = self.buffer + + result.append("static const {}[{}] =\n{{\n".format(self.header, + len(data))) + + for idx in range(len(data)): + result.append(" {},\n".format(data[idx])) + + result.append("};") + + return result + +class SHS: + def __init__(self, data): + self.data = data + + def test(self, idx_from, idx_to): + stat = [] + + for i in range(idx_from, idx_to): + mx = 0 + used = 0 + result = {} + + for entry in self.data: + idx = self.make_id(entry['key'], i) + + if idx not in result: + used += 1 + result[idx] = 0 + + result[idx] += 1 + + if result[idx] > mx: + mx = result[idx] + + stat.append([mx, used, i]) + + stat.sort(key = lambda entr: entr[0]) + best = stat[0] + + print("Max deep {}; Used {} of {}".format(*best)) + + return best[2] + + def make_id(self, key, table_size): + key = key.lower() + return (((ord(key[0]) * ord(key[-1])) + len(key)) % table_size) + 1 + + def make(self, best_size): + self.table_size = best_size + + self.table = [[] for _ in range(self.table_size + 1)] + + for e in self.data: + idx = self.make_id(e['key'], self.table_size) + + self.table[idx].append(e) + self.table[idx].sort(key = lambda entr: len(entr['key'])) + + def build(self): + result = {} + unused = [] + + result[0] = [None, "NULL", self.table_size, 0, True] + + for key in range(1, self.table_size + 1): + if not self.table[key]: + unused.append(key) + continue + + e = self.table[key].pop(0) + result[key] = [e["key"], e["value"], len(e["key"]), 0, True] + + self.idx = self.table_size + self.unused = unused + self.unused_pos = 0 + + for key in range(1, self.table_size + 1): + if not self.table[key]: + continue + + last_entry = result[key] + + for e in self.table[key]: + last_entry[3] = self.next_free_pos() + + new_entry = [e["key"], e["value"], len(e["key"]), 0, False] + + result[last_entry[3]] = new_entry + last_entry = new_entry + + return result + + def next_free_pos(self): + if len(self.unused) > self.unused_pos: + idx = self.unused[self.unused_pos] + self.unused_pos += 1 + return idx + + self.idx += 1 + return self.idx + + +if __name__ == "__main__": + data_name = "njs_lexer_kws" + + def enum(name): + name = re.sub(r"[^a-zA-Z0-9_]", "_", name) + return "NJS_TOKEN_" + name.upper() + + def kw_create(): + t = Table("njs_keyword_t " + data_name) + + for kw in global_keywords: + t.add("{{ .entry = {{ njs_str(\"{}\") }}, .type = {} }}" + .format(kw, enum(kw))) + + return t.create() + + def entries_create(): + shs = SHS([{ "key": kw, + "value": "&{}[{}]".format(data_name, i) } + for i, kw in enumerate(global_keywords)]) + + best_size = shs.test(5, 128) + shs.make(best_size) + lst = shs.build() + + t = Table("njs_lexer_keyword_entry_t njs_lexer_keyword_entries") + + for kw in range(shs.idx + 1): + if kw not in lst: + t.add("{ NULL, NULL, 0, 0 }") + continue + + key_val = "\"{}\"".format(lst[kw][0]) if lst[kw][0] else "NULL" + t.add("{{ {}, {}, {}, {} }}".format(key_val, *lst[kw][1:])) + + return t.create() + + content = [""" +/* + * Copyright (C) Nginx, Inc. + * + * Do not edit, generated by: utils/lexer_keyword.py. + */ + + +#ifndef _NJS_LEXER_TABLES_H_INCLUDED_ +#define _NJS_LEXER_TABLES_H_INCLUDED_ + + +""", + "".join(kw_create()), + "\n\n\n", + "".join(entries_create()), + "\n\n\n#endif /* _NJS_LEXER_TABLES_H_INCLUDED_ */\n"] + + out = "".join(content) + print(out) + + fn = os.path.join(os.path.dirname(__file__), "../src/njs_lexer_tables.h") + + with open(fn, 'w') as fh: + fh.write(out)