From: basavaraj-sm05 Date: Tue, 17 Nov 2026 20:06:15 +0000 (+0530) Subject: Fix surrogate pairs after an unpaired surrogate escape X-Git-Url: https://git.kaiwu.me/http/%7B@url%7D?a=commitdiff_plain;h=b67014d7c41bbf1fb06007122d4061c3bddd97d3;p=njs.git Fix surrogate pairs after an unpaired surrogate escape Previously, when a leading surrogate escape was followed by another leading surrogate escape, JSON.parse() and the string literal parser consumed both and replaced each of them with U+FFFD. A trailing surrogate escape right after them then had no leading half to pair with and was replaced as well, so a valid pair that followed an unpaired leading surrogate was lost. For example, JSON.parse('"\\uD83D\\uD83D\\uDE00"') and "\u{D83D}\u{D83D}\u{DE00}" produced three U+FFFD characters instead of U+FFFD followed by U+1F600. Now only the unpaired leading surrogate is replaced, and the escape after it is decoded on its own, so it can start a new pair. The result matches String.fromCharCode() for the same code units. The issue was introduced in 721af4cc and 5fd752d0 (0.3.3). --- diff --git a/src/njs_json.c b/src/njs_json.c index e6a849a8..1b69c0d0 100644 --- a/src/njs_json.c +++ b/src/njs_json.c @@ -708,22 +708,20 @@ njs_json_parse_string(njs_json_parse_ctx_t *ctx, njs_value_t *value, continue; } - p += 2; + utf_low = njs_json_unicode(p + 2); - utf_low = njs_json_unicode(p); - p += 4; - - if (njs_fast_path(njs_surrogate_trailing(utf_low))) { - utf = njs_surrogate_pair(utf, utf_low); - - } else if (njs_surrogate_leading(utf_low)) { - utf = NJS_UNICODE_REPLACEMENT; - s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT); - - } else { - utf = utf_low; + if (njs_slow_path(!njs_surrogate_trailing(utf_low))) { + /* + * The next escape is left to the next iteration, + * a leading surrogate there can start a new pair. + */ s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT); + continue; } + + p += 6; + + utf = njs_surrogate_pair(utf, utf_low); } s = njs_utf8_encode(s, utf); diff --git a/src/njs_parser.c b/src/njs_parser.c index 53c132d4..b4d42fa4 100644 --- a/src/njs_parser.c +++ b/src/njs_parser.c @@ -8800,18 +8800,14 @@ next_char: if (njs_fast_path(njs_surrogate_trailing(cp))) { cp = njs_surrogate_pair(cp_pair, cp); - } else if (njs_slow_path(njs_surrogate_leading(cp))) { - cp = NJS_UNICODE_REPLACEMENT; - - dst = njs_utf8_encode(dst, (uint32_t) cp); - } else { dst = njs_utf8_encode(dst, NJS_UNICODE_REPLACEMENT); } cp_pair = 0; + } - } else if (njs_surrogate_any(cp)) { + if (njs_surrogate_any(cp)) { if (cp <= 0xdbff && (end - src) >= 2 && src[0] == '\\' && src[1] == 'u') { @@ -8957,20 +8953,15 @@ njs_parser_escape_string_calc_length(njs_parser_t *parser, if (njs_fast_path(njs_surrogate_trailing(cp))) { cp = njs_surrogate_pair(cp_pair, cp); - } else if (njs_slow_path(njs_surrogate_leading(cp))) { - cp = NJS_UNICODE_REPLACEMENT; - - size += njs_utf8_size(cp); - length++; - } else { size += njs_utf8_size(NJS_UNICODE_REPLACEMENT); length++; } cp_pair = 0; + } - } else if (njs_surrogate_any(cp)) { + if (njs_surrogate_any(cp)) { if (cp <= 0xdbff && (end - src) >= 2 && src[0] == '\\' && src[1] == 'u') { diff --git a/src/test/njs_unit_test.c b/src/test/njs_unit_test.c index be382b3e..7f3721fe 100644 --- a/src/test/njs_unit_test.c +++ b/src/test/njs_unit_test.c @@ -8738,6 +8738,12 @@ static njs_unit_test_t njs_test[] = { njs_str("'\\u{d800}\\u{dB00}'"), njs_str("��") }, + { njs_str("'\\ud83d\\ud83d\\ude00'"), + njs_str("\xef\xbf\xbd\xf0\x9f\x98\x80") }, + + { njs_str("'\\u{d83d}\\u{d83d}\\u{de00}'.length"), + njs_str("2") }, + { njs_str("'\\u{d800}\\u{d7ff}'"), njs_str("�퟿") }, @@ -19794,6 +19800,12 @@ static njs_unit_test_t njs_test[] = { njs_str("JSON.parse('\"\\\\uD800\\\\uDB00\"')"), njs_str("��") }, + { njs_str("JSON.parse('\"\\\\uD83D\\\\uD83D\\\\uDE00\"')"), + njs_str("\xef\xbf\xbd\xf0\x9f\x98\x80") }, + + { njs_str("JSON.parse('\"\\\\uD83D\\\\uD83D\\\\uDE00\"').length"), + njs_str("2") }, + { njs_str("JSON.parse('\"\\\\ud800[\"')"), njs_str("�[") },