diff options
Diffstat (limited to 'nginx')
| -rw-r--r-- | nginx/ngx_http_js_module.c | 4 | ||||
| -rw-r--r-- | nginx/ngx_js.h | 5 | ||||
| -rw-r--r-- | nginx/ngx_js_shared_dict.c | 1146 | ||||
| -rw-r--r-- | nginx/ngx_js_shared_dict.h | 1 | ||||
| -rw-r--r-- | nginx/ngx_stream_js_module.c | 4 | ||||
| -rw-r--r-- | nginx/t/js_shared_dict_state.t | 256 | ||||
| -rw-r--r-- | nginx/t/stream_js_shared_dict_state.t | 137 |
7 files changed, 1534 insertions, 19 deletions
diff --git a/nginx/ngx_http_js_module.c b/nginx/ngx_http_js_module.c index 3b493bd1..28798172 100644 --- a/nginx/ngx_http_js_module.c +++ b/nginx/ngx_http_js_module.c @@ -7738,6 +7738,10 @@ ngx_http_js_init_worker(ngx_cycle_t *cycle) return NGX_ERROR; } + if (ngx_js_dict_init_worker(jmcf) != NGX_OK) { + return NGX_ERROR; + } + return NGX_OK; } diff --git a/nginx/ngx_js.h b/nginx/ngx_js.h index ceb82f74..122881af 100644 --- a/nginx/ngx_js.h +++ b/nginx/ngx_js.h @@ -16,8 +16,6 @@ #include <njs.h> #include <njs_rbtree.h> #include <njs_arr.h> -#include "ngx_js_fetch.h" -#include "ngx_js_shared_dict.h" #if (NJS_HAVE_QUICKJS) #include <qjs.h> @@ -440,4 +438,7 @@ extern njs_module_t njs_xml_module; extern njs_module_t njs_zlib_module; +#include "ngx_js_fetch.h" +#include "ngx_js_shared_dict.h" + #endif /* _NGX_JS_H_INCLUDED_ */ diff --git a/nginx/ngx_js_shared_dict.c b/nginx/ngx_js_shared_dict.c index 0a35241f..5445b4f2 100644 --- a/nginx/ngx_js_shared_dict.c +++ b/nginx/ngx_js_shared_dict.c @@ -18,6 +18,9 @@ typedef struct { ngx_rbtree_t rbtree_expire; ngx_rbtree_node_t sentinel_expire; + + unsigned dirty:1; + unsigned writing:1; } ngx_js_dict_sh_t; @@ -26,12 +29,23 @@ struct ngx_js_dict_s { ngx_js_dict_sh_t *sh; ngx_slab_pool_t *shpool; + /** + * in order for ngx_js_dict_t to be used as a ngx_event_t data, + * fd is used for event debug and should be at the same position + * as in ngx_connection_t. see ngx_event_ident() for details. + */ + ngx_socket_t fd; + ngx_msec_t timeout; ngx_flag_t evict; #define NGX_JS_DICT_TYPE_STRING 0 #define NGX_JS_DICT_TYPE_NUMBER 1 ngx_uint_t type; + ngx_event_t save_event; + ngx_str_t state_file; + ngx_str_t state_temp_file; + ngx_js_dict_t *next; }; @@ -49,6 +63,13 @@ typedef struct { } ngx_js_dict_node_t; +typedef struct { + ngx_str_t key; + ngx_js_dict_value_t value; + ngx_msec_t expire; +} ngx_js_dict_entry_t; + + static njs_int_t njs_js_ext_shared_dict_capacity(njs_vm_t *vm, njs_object_prop_t *prop, uint32_t unused, njs_value_t *value, njs_value_t *setval, njs_value_t *retval); @@ -625,8 +646,14 @@ njs_js_ext_shared_dict_clear(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, done: + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + njs_value_undefined_set(retval); return NJS_OK; @@ -1246,6 +1273,16 @@ njs_js_ext_shared_dict_type(njs_vm_t *vm, njs_object_prop_t *prop, } +static njs_int_t +ngx_js_dict_shared_error_name(njs_vm_t *vm, njs_object_prop_t *prop, + uint32_t unused, njs_value_t *value, njs_value_t *setval, + njs_value_t *retval) +{ + return njs_vm_value_string_create(vm, retval, + (u_char *) "SharedMemoryError", 17); +} + + static ngx_js_dict_node_t * ngx_js_dict_lookup(ngx_js_dict_t *dict, ngx_str_t *key) { @@ -1329,8 +1366,14 @@ ngx_js_dict_set(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_str_t *key, } } + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return NGX_OK; memory_error: @@ -1494,8 +1537,14 @@ ngx_js_dict_delete(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_str_t *key, ngx_js_dict_node_free(dict, node); + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return rc; } @@ -1536,8 +1585,14 @@ ngx_js_dict_incr(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_str_t *key, } } + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return NGX_OK; } @@ -1676,13 +1731,1013 @@ ngx_js_dict_evict(ngx_js_dict_t *dict, ngx_int_t count) } -static njs_int_t -ngx_js_dict_shared_error_name(njs_vm_t *vm, njs_object_prop_t *prop, - uint32_t unused, njs_value_t *value, njs_value_t *setval, - njs_value_t *retval) +static ngx_int_t +ngx_js_render_string(njs_chb_t *chain, ngx_str_t *str) { - return njs_vm_value_string_create(vm, retval, - (u_char *) "SharedMemoryError", 17); + size_t size; + u_char c, *dst, *dst_end; + const u_char *p, *end; + + static char hex2char[16] = { '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + + p = str->data; + end = p + str->len; + size = str->len + 2; + + dst = njs_chb_reserve(chain, size); + if (dst == NULL) { + return NGX_ERROR; + } + + dst_end = dst + size; + + *dst++ = '\"'; + njs_chb_written(chain, 1); + + while (p < end) { + if (dst_end <= dst + sizeof("\\uXXXX")) { + size = ngx_max(end - p + 1, 6); + dst = njs_chb_reserve(chain, size); + if (dst == NULL) { + return NGX_ERROR; + } + + dst_end = dst + size; + } + + if (*p < ' ' || *p == '\\' || *p == '\"') { + c = (u_char) *p++; + *dst++ = '\\'; + njs_chb_written(chain, 2); + + switch (c) { + case '\\': + *dst++ = '\\'; + break; + case '"': + *dst++ = '\"'; + break; + case '\r': + *dst++ = 'r'; + break; + case '\n': + *dst++ = 'n'; + break; + case '\t': + *dst++ = 't'; + break; + case '\b': + *dst++ = 'b'; + break; + case '\f': + *dst++ = 'f'; + break; + default: + *dst++ = 'u'; + *dst++ = '0'; + *dst++ = '0'; + *dst++ = hex2char[(c & 0xf0) >> 4]; + *dst++ = hex2char[c & 0x0f]; + njs_chb_written(chain, 4); + } + + continue; + } + + dst = njs_utf8_copy(dst, &p, end); + + njs_chb_written(chain, dst - chain->last->pos); + } + + njs_chb_append_literal(chain, "\""); + + return NGX_OK; +} + + +static ngx_int_t +ngx_js_dict_render_json(ngx_js_dict_t *dict, njs_chb_t *chain) +{ + u_char *p, *dst; + size_t len; + ngx_msec_t now; + ngx_time_t *tp; + ngx_rbtree_t *rbtree; + ngx_rbtree_node_t *rn, *next; + ngx_js_dict_node_t *node; + + tp = ngx_timeofday(); + now = tp->sec * 1000 + tp->msec; + + rbtree = &dict->sh->rbtree; + + njs_chb_append_literal(chain,"{"); + + if (rbtree->root == rbtree->sentinel) { + njs_chb_append_literal(chain, "}"); + return NGX_OK; + } + + for (rn = ngx_rbtree_min(rbtree->root, rbtree->sentinel); + rn != NULL; + rn = next) + { + node = (ngx_js_dict_node_t *) rn; + + next = ngx_rbtree_next(rbtree, rn); + + if (dict->timeout && now >= node->expire.key) { + continue; + } + + if (ngx_js_render_string(chain, &node->sn.str) != NGX_OK) { + return NGX_ERROR; + } + + njs_chb_append_literal(chain,":{"); + + if (dict->type == NGX_JS_DICT_TYPE_STRING) { + njs_chb_append_literal(chain,"\"value\":"); + + if (ngx_js_render_string(chain, &node->value.str) != NGX_OK) { + return NGX_ERROR; + } + + } else { + len = sizeof("\"value\":.") + 18 + 6; + dst = njs_chb_reserve(chain, len); + if (dst == NULL) { + return NGX_ERROR; + } + + p = njs_sprintf(dst, dst + len, "\"value\":%.6f", + node->value.number); + njs_chb_written(chain, p - dst); + } + + if (dict->timeout) { + len = sizeof(",\"expire\":1000000000"); + dst = njs_chb_reserve(chain, len); + if (dst == NULL) { + return NGX_ERROR; + } + + p = njs_sprintf(dst, dst + len, ",\"expire\":%ui", + node->expire.key); + njs_chb_written(chain, p - dst); + } + + njs_chb_append_literal(chain, "}"); + + if (next != NULL) { + njs_chb_append_literal(chain, ","); + } + } + + njs_chb_append_literal(chain, "}"); + + return NGX_OK; +} + + +static u_char * +ngx_js_skip_space(u_char *start, u_char *end) +{ + u_char *p; + + for (p = start; p != end; p++) { + + switch (*p) { + case ' ': + case '\t': + case '\r': + case '\n': + continue; + } + + break; + } + + return p; +} + + +static uint32_t +ngx_js_unicode(const u_char *p) +{ + u_char c; + uint32_t utf; + njs_uint_t i; + + utf = 0; + + for (i = 0; i < 4; i++) { + utf <<= 4; + c = p[i] | 0x20; + c -= '0'; + if (c > 9) { + c += '0' - 'a' + 10; + } + + utf |= c; + } + + return utf; +} + + +static u_char * +ngx_js_dict_parse_string(ngx_pool_t *pool, u_char *p, u_char *end, + ngx_str_t *str, const char **err, u_char **at) +{ + u_char ch, *s, *dst, *start, *last; + size_t size, surplus; + uint32_t utf, utf_low; + + enum { + sw_usual = 0, + sw_escape, + sw_encoded1, + sw_encoded2, + sw_encoded3, + sw_encoded4, + } state; + + if (*p != '"') { + *err = "unexpected character, expected '\"'"; + goto error; + } + + start = p + 1; + + dst = NULL; + state = 0; + surplus = 0; + + for (p = start; p < end; p++) { + ch = *p; + + switch (state) { + + case sw_usual: + + if (ch == '"') { + break; + } + + if (ch == '\\') { + state = sw_escape; + continue; + } + + if (ch >= ' ') { + continue; + } + + *err = "Invalid source char"; + goto error; + + case sw_escape: + + switch (ch) { + case '"': + case '\\': + case '/': + case 'n': + case 'r': + case 't': + case 'b': + case 'f': + surplus++; + state = sw_usual; + continue; + + case 'u': + /* + * Basic unicode 6 bytes "\uXXXX" in JSON + * and up to 3 bytes in UTF-8. + * + * Surrogate pair: 12 bytes "\uXXXX\uXXXX" in JSON + * and 3 or 4 bytes in UTF-8. + */ + surplus += 3; + state = sw_encoded1; + continue; + } + + *err = "Invalid escape char"; + goto error; + + case sw_encoded1: + case sw_encoded2: + case sw_encoded3: + case sw_encoded4: + + if ((ch >= '0' && ch <= '9') + || (ch >= 'A' && ch <= 'F') + || (ch >= 'a' && ch <= 'f')) + { + state = (state == sw_encoded4) ? sw_usual : state + 1; + continue; + } + + *err = "Invalid Unicode escape sequence"; + goto error; + } + + break; + } + + if (p == end) { + *err = "unexpected end of input"; + goto error; + } + + /* Points to the ending quote mark. */ + last = p; + + size = last - start - surplus; + + if (surplus != 0) { + p = start; + + dst = ngx_palloc(pool, size); + if (dst == NULL) { + *err = "out of memory"; + goto error; + } + + s = dst; + + do { + ch = *p++; + + if (ch != '\\') { + *s++ = ch; + continue; + } + + ch = *p++; + + switch (ch) { + case '"': + case '\\': + case '/': + *s++ = ch; + continue; + + case 'n': + *s++ = '\n'; + continue; + + case 'r': + *s++ = '\r'; + continue; + + case 't': + *s++ = '\t'; + continue; + + case 'b': + *s++ = '\b'; + continue; + + case 'f': + *s++ = '\f'; + continue; + } + + /* "\uXXXX": Unicode escape sequence. */ + + utf = ngx_js_unicode(p); + p += 4; + + if (njs_surrogate_any(utf)) { + + if (utf > 0xdbff || p[0] != '\\' || p[1] != 'u') { + s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT); + continue; + } + + p += 2; + + utf_low = ngx_js_unicode(p); + p += 4; + + if (njs_fast_path(njs_surrogate_trailing(utf_low))) { + utf = njs_surrogate_pair(utf, utf_low); + + } else if (njs_surrogate_leading(utf_low)) { + utf = NJS_UNICODE_REPLACEMENT; + s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT); + + } else { + utf = utf_low; + s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT); + } + } + + s = njs_utf8_encode(s, utf); + + } while (p != last); + + size = s - dst; + start = dst; + } + + str->data = start; + str->len = size; + + return p + 1; + +error: + + *at = p; + + return NULL; +} + + +static u_char * +ngx_js_dict_parse_entry(ngx_js_dict_t *dict, ngx_pool_t *pool, + ngx_js_dict_entry_t *entry, u_char *buf, u_char *end, const char **err, + u_char **at) +{ + int see_value; + u_char *p, *pp; + double number; + ngx_str_t key, str; + + p = buf; + + if (*p++ != '{') { + *err = "unexpected character, expected '{'"; + goto error; + } + + see_value = 0; + + while (1) { + p = ngx_js_skip_space(p, end); + if (p == end) { + *err = "unexpected end of json"; + goto error; + } + + if (*p == '}') { + break; + } + + p = ngx_js_dict_parse_string(pool, p, end, &key, err, at); + if (p == NULL) { + return NULL; + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + *err = "unexpected end of json"; + goto error; + } + + if (*p++ != ':') { + *err = "unexpected character, expected ':'"; + goto error; + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + *err = "unexpected end of json"; + goto error; + } + + if (*p == '\"') { + p = ngx_js_dict_parse_string(pool, p, end, &str, err, at); + if (p == NULL) { + return NULL; + } + + if (key.len == 5 && ngx_strncmp(key.data, "value", 5) == 0) { + if (dict->type != NGX_JS_DICT_TYPE_STRING) { + *err = "expected string value"; + goto error; + } + + entry->value.str = str; + see_value = 1; + } + + } else { + pp = p; + number = strtod((char *) p, (char **) &p); + if (pp == p) { + *err = "invalid number value"; + goto error; + } + + if (key.len == 5 && ngx_strncmp(key.data, "value", 5) == 0) { + if (dict->type == NGX_JS_DICT_TYPE_STRING) { + *err = "expected number value"; + goto error; + } + + entry->value.number = number; + see_value = 1; + + } else if (key.len == 6 + && ngx_strncmp(key.data, "expire", 6) == 0) + { + entry->expire = number; + } + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + *err = "unexpected end of json"; + goto error; + } + + if (*p == ',') { + p++; + } + } + + if (!see_value) { + *err = "missing value"; + goto error; + } + + return p + 1; + +error: + + *at = p; + + return NULL; +} + + +static ngx_int_t +ngx_js_dict_parse_state(ngx_js_dict_t *dict, ngx_pool_t *pool, + ngx_array_t *entries, u_char *buf, u_char *end) +{ + u_char *p, *at; + const char *err; + ngx_js_dict_entry_t *e; + + /* GCC complains about uninitialized err, at. */ + + err = ""; + at = NULL; + + p = ngx_js_skip_space(buf, end); + if (p == end) { + err = "empty json"; + goto error; + } + + if (*p++ != '{') { + err = "json must start with '{'"; + goto error; + } + + while (1) { + p = ngx_js_skip_space(p, end); + if (p == end) { + err = "unexpected end of json"; + goto error; + } + + if (*p == '}') { + p++; + break; + } + + e = ngx_array_push(entries); + if (e == NULL) { + return NGX_ERROR; + } + + p = ngx_js_dict_parse_string(pool, p, end, &e->key, &err, &at); + if (p == NULL) { + p = at; + goto error; + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + err = "unexpected end of json"; + goto error; + } + + if (*p++ != ':') { + err = "unexpected character, expected ':'"; + goto error; + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + err = "unexpected end of json"; + goto error; + } + + p = ngx_js_dict_parse_entry(dict, pool, e, p, end, &err, &at); + if (p == NULL) { + p = at; + goto error; + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + err = "unexpected end of json"; + goto error; + } + + if (*p == ',') { + p++; + } + } + + p = ngx_js_skip_space(p, end); + + if (p != end) { + err = "unexpected character, expected end of json"; + goto error; + } + + return NGX_OK; + +error: + + ngx_log_error(NGX_LOG_EMERG, dict->shm_zone->shm.log, 0, + "invalid format while loading js_shared_dict_zone \"%V\"" + " from state file \"%s\": %s at offset %z", + &dict->shm_zone->shm.name, dict->state_file.data, err, + p - buf); + + return NGX_ERROR; +} + + +static ngx_int_t +ngx_js_dict_save(ngx_js_dict_t *dict) +{ + + u_char *name; + ngx_int_t rc; + ngx_log_t *log; + njs_chb_t chain; + ngx_file_t file; + ngx_pool_t *pool; + ngx_chain_t *out, *cl, **ll; + njs_chb_node_t *node; + ngx_ext_rename_file_t ext; + + log = dict->shm_zone->shm.log; + + pool = ngx_create_pool(NGX_DEFAULT_POOL_SIZE, log); + if (pool == NULL) { + return NGX_ERROR; + } + + ngx_rwlock_wlock(&dict->sh->rwlock); + + if (!dict->sh->dirty) { + ngx_rwlock_unlock(&dict->sh->rwlock); + ngx_destroy_pool(pool); + return NGX_OK; + } + + if (dict->sh->writing) { + ngx_rwlock_unlock(&dict->sh->rwlock); + ngx_destroy_pool(pool); + return NGX_AGAIN; + } + + ngx_rwlock_downgrade(&dict->sh->rwlock); + + NGX_CHB_CTX_INIT(&chain, pool); + + rc = ngx_js_dict_render_json(dict, &chain); + + if (rc != NGX_OK) { + ngx_rwlock_unlock(&dict->sh->rwlock); + ngx_destroy_pool(pool); + return rc; + } + + dict->sh->writing = 1; + dict->sh->dirty = 0; + + ngx_rwlock_unlock(&dict->sh->rwlock); + + name = dict->state_temp_file.data; + + out = NULL; + ll = &out; + + for (node = chain.nodes; node != NULL; node = node->next) { + cl = ngx_alloc_chain_link(pool); + if (cl == NULL) { + goto error; + } + + cl->buf = ngx_calloc_buf(pool); + if (cl->buf == NULL) { + goto error; + } + + cl->buf->pos = node->start; + cl->buf->last = node->pos; + cl->buf->memory = 1; + cl->buf->last_buf = (node->next == NULL) ? 1 : 0; + + *ll = cl; + ll = &cl->next; + } + + *ll = NULL; + + ngx_memzero(&file, sizeof(ngx_file_t)); + file.name = dict->state_temp_file; + file.log = log; + + file.fd = ngx_open_file(file.name.data, NGX_FILE_WRONLY, NGX_FILE_TRUNCATE, + NGX_FILE_DEFAULT_ACCESS); + + if (file.fd == NGX_INVALID_FILE) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + ngx_open_file_n " \"%s\" failed", name); + goto error; + } + + rc = ngx_write_chain_to_file(&file, out, 0, pool); + + if (rc == NGX_ERROR) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + ngx_write_fd_n " \"%s\" failed", file.name.data); + goto error; + } + + if (ngx_close_file(file.fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", file.name.data); + } + + file.fd = NGX_INVALID_FILE; + + ext.access = 0; + ext.time = -1; + ext.create_path = 0; + ext.delete_file = 0; + ext.log = log; + + if (ngx_ext_rename_file(&dict->state_temp_file, &dict->state_file, &ext) + != NGX_OK) + { + goto error; + } + + /* no lock required */ + dict->sh->writing = 0; + ngx_destroy_pool(pool); + + return NGX_OK; + +error: + + if (file.fd != NGX_INVALID_FILE + && ngx_close_file(file.fd) == NGX_FILE_ERROR) + { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", name); + } + + ngx_destroy_pool(pool); + + /* no lock required */ + dict->sh->writing = 0; + dict->sh->dirty = 1; + + return NGX_ERROR; +} + + +static ngx_int_t +ngx_js_dict_load(ngx_js_dict_t *dict) +{ + off_t size; + u_char *name, *buf; + size_t len; + ssize_t n; + ngx_fd_t fd; + ngx_err_t err; + ngx_int_t rc; + ngx_log_t *log; + ngx_uint_t i; + ngx_msec_t now, expire; + ngx_time_t *tp; + ngx_pool_t *pool; + ngx_array_t data; + ngx_file_info_t fi; + ngx_js_dict_entry_t *entries; + + if (dict->state_file.data == NULL) { + return NGX_OK; + } + + log = dict->shm_zone->shm.log; + + name = dict->state_file.data; + + fd = ngx_open_file(name, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); + if (fd == NGX_INVALID_FILE) { + err = ngx_errno; + + if (err == NGX_ENOENT || err == NGX_ENOPATH) { + return NGX_OK; + } + + ngx_log_error(NGX_LOG_EMERG, log, err, + ngx_open_file_n " \"%s\" failed", name); + return NGX_ERROR; + } + + if (ngx_fd_info(fd, &fi) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + ngx_fd_info_n " \"%s\" failed", name); + pool = NULL; + goto failed; + } + + size = ngx_file_size(&fi); + + if (size == 0) { + + if (ngx_close_file(fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", name); + } + + return NGX_OK; + } + + pool = ngx_create_pool(NGX_DEFAULT_POOL_SIZE, log); + if (pool == NULL) { + goto failed; + } + + len = size; + + buf = ngx_pnalloc(pool, len); + if (buf == NULL) { + goto failed; + } + + n = ngx_read_fd(fd, buf, len); + + if (n == -1) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + ngx_read_fd_n " \"%s\" failed", name); + goto failed; + } + + if ((size_t) n != len) { + ngx_log_error(NGX_LOG_EMERG, log, 0, + ngx_read_fd_n " has read only %z of %uz from %s", + n, len, name); + goto failed; + } + + if (ngx_close_file(fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", name); + fd = NGX_INVALID_FILE; + goto failed; + } + + fd = NGX_INVALID_FILE; + + if (ngx_array_init(&data, pool, 4, sizeof(ngx_js_dict_entry_t)) + != NGX_OK) + { + goto failed; + } + + rc = ngx_js_dict_parse_state(dict, pool, &data, buf, buf + len); + + if (rc != NGX_OK) { + goto failed; + } + + entries = data.elts; + + tp = ngx_timeofday(); + now = tp->sec * 1000 + tp->msec; + + for (i = 0; i < data.nelts; i++) { + + if (dict->timeout) { + expire = entries[i].expire; + + if (expire && now >= expire) { + dict->sh->dirty = 1; + continue; + } + + if (expire == 0) { + /* treat state without expire as new */ + expire = now + dict->timeout; + dict->sh->dirty = 1; + } + + } else { + expire = 0; + } + + if (ngx_js_dict_lookup(dict, &entries[i].key) != NULL) { + goto failed; + } + + if (ngx_js_dict_add_value(dict, &entries[i].key, &entries[i].value, + expire, 1) + != NGX_OK) + { + goto failed; + } + } + + ngx_destroy_pool(pool); + + return NGX_OK; + +failed: + + if (fd != NGX_INVALID_FILE && ngx_close_file(fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", name); + } + + if (pool) { + ngx_destroy_pool(pool); + } + + return NGX_ERROR; +} + + +static void +ngx_js_dict_save_handler(ngx_event_t *ev) +{ + ngx_int_t rc; + ngx_js_dict_t *dict; + + dict = ev->data; + + rc = ngx_js_dict_save(dict); + + if (rc == NGX_OK) { + return; + } + + if (rc == NGX_ERROR && (ngx_terminate || ngx_exiting)) { + ngx_log_error(NGX_LOG_ALERT, ev->log, 0, + "failed to save the state of shared dict zone \"%V\"", + &dict->shm_zone->shm.name); + return; + } + + /* NGX_ERROR, NGX_AGAIN */ + + ngx_add_timer(ev, 1000); +} + + +ngx_int_t +ngx_js_dict_init_worker(ngx_js_main_conf_t *jmcf) +{ + ngx_js_dict_t *dict; + + if ((ngx_process != NGX_PROCESS_WORKER || ngx_worker != 0) + && ngx_process != NGX_PROCESS_SINGLE) + { + return NGX_OK; + } + + if (jmcf->dicts == NULL) { + return NGX_OK; + } + + for (dict = jmcf->dicts; dict != NULL; dict = dict->next) { + + if (!dict->sh->dirty || !dict->state_file.data) { + continue; + } + + ngx_add_timer(&dict->save_event, 1000); + } + + return NGX_OK; } @@ -1752,6 +2807,10 @@ ngx_js_dict_init_zone(ngx_shm_zone_t *shm_zone, void *data) ngx_sprintf(dict->shpool->log_ctx, " in js shared zone \"%V\"%Z", &shm_zone->shm.name); + if (ngx_js_dict_load(dict) != NGX_OK) { + return NGX_ERROR; + } + return NGX_OK; } @@ -1764,7 +2823,7 @@ ngx_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, u_char *p; ssize_t size; - ngx_str_t *value, name, s; + ngx_str_t *value, name, file, s; ngx_flag_t evict; ngx_msec_t timeout; ngx_uint_t i, type; @@ -1775,6 +2834,7 @@ ngx_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, evict = 0; timeout = 0; name.len = 0; + ngx_str_null(&file); type = NGX_JS_DICT_TYPE_STRING; value = cf->args->elts; @@ -1826,6 +2886,17 @@ ngx_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, continue; } + if (ngx_strncmp(value[i].data, "state=", 6) == 0) { + file.data = value[i].data + 6; + file.len = value[i].len - 6; + + if (ngx_conf_full_name(cf->cycle, &file, 0) != NGX_OK) { + return NGX_CONF_ERROR; + } + + continue; + } + if (ngx_strncmp(value[i].data, "timeout=", 8) == 0) { s.data = value[i].data + 8; @@ -1899,6 +2970,23 @@ ngx_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, dict->timeout = timeout; dict->type = type; + dict->save_event.handler = ngx_js_dict_save_handler; + dict->save_event.data = dict; + dict->save_event.log = &cf->cycle->new_log; + dict->fd = -1; + + if (file.data) { + dict->state_file = file; + + p = ngx_pnalloc(cf->pool, file.len + sizeof(".tmp")); + if (p == NULL) { + return NGX_CONF_ERROR; + } + + dict->state_temp_file.data = p; + dict->state_temp_file.len = ngx_sprintf(p, "%V.tmp%Z", &file) - p - 1; + } + return NGX_CONF_OK; } @@ -2098,8 +3186,14 @@ ngx_qjs_ext_shared_dict_clear(JSContext *cx, JSValueConst this_val, done: + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return JS_UNDEFINED; } @@ -2746,8 +3840,14 @@ ngx_qjs_dict_delete(JSContext *cx, ngx_js_dict_t *dict, ngx_str_t *key, ngx_js_dict_node_free(dict, node); + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return ret; } @@ -2825,8 +3925,14 @@ ngx_qjs_dict_incr(JSContext *cx, ngx_js_dict_t *dict, ngx_str_t *key, } } + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return value; } @@ -2856,24 +3962,30 @@ ngx_qjs_dict_set(JSContext *cx, ngx_js_dict_t *dict, ngx_str_t *key, goto memory_error; } - ngx_rwlock_unlock(&dict->sh->rwlock); + } else { - return JS_TRUE; - } + if (flags & NGX_JS_DICT_FLAG_MUST_NOT_EXIST) { + if (!dict->timeout || now < node->expire.key) { + ngx_rwlock_unlock(&dict->sh->rwlock); + return JS_FALSE; + } + } - if (flags & NGX_JS_DICT_FLAG_MUST_NOT_EXIST) { - if (!dict->timeout || now < node->expire.key) { - ngx_rwlock_unlock(&dict->sh->rwlock); - return JS_FALSE; + if (ngx_qjs_dict_update(cx, dict, node, value, timeout, now) + != NGX_OK) + { + goto memory_error; } } - if (ngx_qjs_dict_update(cx, dict, node, value, timeout, now) != NGX_OK) { - goto memory_error; - } + dict->sh->dirty = 1; ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return JS_TRUE; memory_error: diff --git a/nginx/ngx_js_shared_dict.h b/nginx/ngx_js_shared_dict.h index b9c7f967..b082962c 100644 --- a/nginx/ngx_js_shared_dict.h +++ b/nginx/ngx_js_shared_dict.h @@ -13,6 +13,7 @@ njs_int_t njs_js_ext_global_shared_prop(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *retval); njs_int_t njs_js_ext_global_shared_keys(njs_vm_t *vm, njs_value_t *value, njs_value_t *keys); +ngx_int_t ngx_js_dict_init_worker(ngx_js_main_conf_t *jmcf); extern njs_module_t ngx_js_shared_dict_module; diff --git a/nginx/ngx_stream_js_module.c b/nginx/ngx_stream_js_module.c index bc653a5b..fb58cdc6 100644 --- a/nginx/ngx_stream_js_module.c +++ b/nginx/ngx_stream_js_module.c @@ -3246,6 +3246,10 @@ ngx_stream_js_init_worker(ngx_cycle_t *cycle) return NGX_ERROR; } + if (ngx_js_dict_init_worker(jmcf) != NGX_OK) { + return NGX_ERROR; + } + return NGX_OK; } diff --git a/nginx/t/js_shared_dict_state.t b/nginx/t/js_shared_dict_state.t new file mode 100644 index 00000000..32eef948 --- /dev/null +++ b/nginx/t/js_shared_dict_state.t @@ -0,0 +1,256 @@ +#!/usr/bin/perl + +# (C) Dmitry Volyntsev +# (C) Nginx, Inc. + +# Tests for js_shared_dict_zone directive, state= parameter. + +############################################################################### + +use warnings; +use strict; + +use Test::More; +use Socket qw/ CRLF /; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +eval { require JSON::PP; }; +plan(skip_all => "JSON::PP not installed") if $@; + +my $t = Test::Nginx->new()->has(qw/http/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + js_import test.js; + + js_shared_dict_zone zone=bar:64k type=string state=bar.json; + js_shared_dict_zone zone=waka:32k timeout=1000s type=number state=waka.json; + + server { + listen 127.0.0.1:8080; + server_name localhost; + + location /add { + js_content test.add; + } + + location /clear { + js_content test.clear; + } + + location /delete { + js_content test.del; + } + + location /get { + js_content test.get; + } + + location /incr { + js_content test.incr; + } + + location /pop { + js_content test.pop; + } + + location /set { + js_content test.set; + } + } +} + +EOF + +$t->write_file('bar.json', <<EOF); +{"waka":{"value":"foo","expire":0}, + "bar": { "value" :"\\u0061\\u0062\\u0063"}, +"FOO \\n": { "value" : "BAZ", "unexpected_str": "u\\r" }, + "X": { "valu\\u0065" : "\\n" , "unexpected_num": 23.1 } , + "\\u0061\\u0062\\u0063": { "value" : "def" } , +} +EOF + +$t->write_file('test.js', <<'EOF'); + function convertToValue(dict, v) { + if (dict.type == 'number') { + return parseInt(v); + + } else if (v == 'empty') { + v = ''; + } + + return v; + } + + function add(r) { + var dict = ngx.shared[r.args.dict]; + var value = convertToValue(dict, r.args.value); + + if (r.args.timeout) { + var timeout = Number(r.args.timeout); + r.return(200, dict.add(r.args.key, value, timeout)); + + } else { + r.return(200, dict.add(r.args.key, value)); + } + } + + function clear(r) { + var dict = ngx.shared[r.args.dict]; + var result = dict.clear(); + r.return(200, result === undefined ? 'undefined' : result); + } + + function del(r) { + var dict = ngx.shared[r.args.dict]; + r.return(200, dict.delete(r.args.key)); + } + + function get(r) { + var dict = ngx.shared[r.args.dict]; + var val = dict.get(r.args.key); + + if (val == '') { + val = 'empty'; + + } else if (val === undefined) { + val = 'undefined'; + } + + r.return(200, val); + } + + function incr(r) { + var dict = ngx.shared[r.args.dict]; + var def = r.args.def ? parseInt(r.args.def) : 0; + + if (r.args.timeout) { + var timeout = Number(r.args.timeout); + var val = dict.incr(r.args.key, parseInt(r.args.by), def, timeout); + r.return(200, val); + + } else { + var val = dict.incr(r.args.key, parseInt(r.args.by), def); + r.return(200, val); + } + } + + function pop(r) { + var dict = ngx.shared[r.args.dict]; + var val = dict.pop(r.args.key); + if (val == '') { + val = 'empty'; + + } else if (val === undefined) { + val = 'undefined'; + } + + r.return(200, val); + } + + function set(r) { + var dict = ngx.shared[r.args.dict]; + var value = convertToValue(dict, r.args.value); + + if (r.args.timeout) { + var timeout = Number(r.args.timeout); + r.return(200, dict.set(r.args.key, value, timeout) === dict); + + } else { + r.return(200, dict.set(r.args.key, value) === dict); + } + } + + export default { add, clear, del, get, incr, pop, set }; +EOF + +$t->try_run('no js_shared_dict_zone with state=')->plan(11); + +############################################################################### + +like(http_get('/get?dict=bar&key=waka'), qr/foo/, 'get bar.waka'); +like(http_get('/get?dict=bar&key=bar'), qr/abc/, 'get bar.bar'); +like(http_get('/get?dict=bar&key=FOO%20%0A'), qr/BAZ/, 'get bar["FOO \\n"]'); +like(http_get('/get?dict=bar&key=abc'), qr/def/, 'get bar.abc'); + +http_get('/set?dict=bar&key=waka&value=foo2'); +http_get('/delete?dict=bar&key=bar'); + +http_get('/set?dict=waka&key=foo&value=42'); + +select undef, undef, undef, 1.1; + +$t->reload(); + +my $bar_state = read_state($t, 'bar.json'); +my $waka_state = read_state($t, 'waka.json'); + +is($bar_state->{waka}->{value}, 'foo2', 'get bar.waka from state'); +is($bar_state->{bar}, undef, 'no bar.bar in state'); +is($waka_state->{foo}->{value}, '42', 'get waka.foo from state'); +like($waka_state->{foo}->{expire}, qr/^\d+$/, 'waka.foo expire'); + +http_get('/pop?dict=bar&key=FOO%20%0A'); + +http_get('/incr?dict=waka&key=foo&by=1'); + +select undef, undef, undef, 1.1; + +$bar_state = read_state($t, 'bar.json'); +$waka_state = read_state($t, 'waka.json'); + +is($bar_state->{'FOO \\n'}, undef, 'no bar.FOO \\n in state'); +is($waka_state->{foo}->{value}, '43', 'get waka.foo from state'); + +http_get('/clear?dict=bar'); + +select undef, undef, undef, 1.1; + +$bar_state = read_state($t, 'bar.json'); + +is($bar_state->{waka}, undef, 'no bar.waka in state'); + +############################################################################### + +sub decode_json { + my $json; + eval { $json = JSON::PP::decode_json(shift) }; + + if ($@) { + return "<failed to parse JSON>"; + } + + return $json; +} + +sub read_state { + my ($self, $file) = @_; + my $json = $self->read_file($file); + + if ($json) { + $json = decode_json($json); + } + + return $json; +} + +############################################################################### diff --git a/nginx/t/stream_js_shared_dict_state.t b/nginx/t/stream_js_shared_dict_state.t new file mode 100644 index 00000000..c2edb63e --- /dev/null +++ b/nginx/t/stream_js_shared_dict_state.t @@ -0,0 +1,137 @@ +#!/usr/bin/perl + +# (C) Dmitry Volyntsev +# (C) Nginx, Inc. + +# Tests for js_shared_dict_zone directive, state= parameter. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; +use Test::Nginx::Stream qw/ stream /; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/stream/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +stream { + %%TEST_GLOBALS_STREAM%% + + js_import test.js; + + js_shared_dict_zone zone=foo:32k state=foo.json; + + server { + listen 127.0.0.1:8081; + js_preread test.preread_verify; + proxy_pass 127.0.0.1:8090; + } +} + +EOF + +$t->write_file('foo.json', <<EOF); +{"QZ":{"value":"1"},"QQ":{"value":"1"}} +EOF + +$t->write_file('test.js', <<EOF); + function preread_verify(s) { + var collect = Buffer.from([]); + + s.on('upstream', function (data, flags) { + collect = Buffer.concat([collect, data]); + + if (collect.length >= 4 && collect.readUInt16BE(0) == 0xabcd) { + let id = collect.slice(2,4); + + ngx.shared.foo.get(id) ? s.done(): s.deny(); + + } else if (collect.length) { + s.deny(); + } + }); + } + + export default { preread_verify }; + +EOF + +$t->try_run('no js_shared_dict_zone with state='); + +$t->plan(2); + +$t->run_daemon(\&stream_daemon, port(8090)); +$t->waitforsocket('127.0.0.1:' . port(8090)); + +############################################################################### + +is(stream('127.0.0.1:' . port(8081))->io("\xAB\xCDQY##"), "", + 'access failed, QY is not in the shared dict'); +is(stream('127.0.0.1:' . port(8081))->io("\xAB\xCDQZ##"), "\xAB\xCDQZ##", + 'access granted'); + +############################################################################### + +sub stream_daemon { + my $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalAddr => '127.0.0.1:' . port(8090), + Listen => 5, + Reuse => 1 + ) + or die "Can't create listening socket: $!\n"; + + local $SIG{PIPE} = 'IGNORE'; + + while (my $client = $server->accept()) { + $client->autoflush(1); + + log2c("(new connection $client)"); + + $client->sysread(my $buffer, 65536) or next; + + log2i("$client $buffer"); + + log2o("$client $buffer"); + + $client->syswrite($buffer); + + close $client; + } +} + +sub log2i { Test::Nginx::log_core('|| <<', @_); } +sub log2o { Test::Nginx::log_core('|| >>', @_); } +sub log2c { Test::Nginx::log_core('||', @_); } + +sub get { + my ($url, %extra) = @_; + + my $s = IO::Socket::INET->new( + Proto => 'tcp', + PeerAddr => '127.0.0.1:' . port(8082) + ) or die "Can't connect to nginx: $!\n"; + + return http_get($url, socket => $s); +} + +############################################################################### |