]> git.kaiwu.me - njs.git/commitdiff
Fix surrogate pairs after an unpaired surrogate escape
authorbasavaraj-sm05 <basavaraj@digiscrypt.com>
Tue, 17 Nov 2026 20:06:15 +0000 (01:36 +0530)
committerDmitry Volyntsev <xeioexception@gmail.com>
Wed, 18 Nov 2026 23:37:16 +0000 (15:37 -0800)
Previously, when a leading surrogate escape was followed by another
leading surrogate escape, JSON.parse() and the string literal parser
consumed both and replaced each of them with U+FFFD.  A trailing
surrogate escape right after them then had no leading half to pair
with and was replaced as well, so a valid pair that followed an
unpaired leading surrogate was lost.  For example,
JSON.parse('"\\uD83D\\uD83D\\uDE00"') and "\u{D83D}\u{D83D}\u{DE00}"
produced three U+FFFD characters instead of U+FFFD followed by
U+1F600.

Now only the unpaired leading surrogate is replaced, and the escape
after it is decoded on its own, so it can start a new pair.  The
result matches String.fromCharCode() for the same code units.

The issue was introduced in 721af4cc and 5fd752d0 (0.3.3).

src/njs_json.c
src/njs_parser.c
src/test/njs_unit_test.c

index e6a849a81a0cd9293e17d811e07aeae32d77596f..1b69c0d0bb977290b8f12acf7df9be0b75a57d35 100644 (file)
@@ -708,22 +708,20 @@ njs_json_parse_string(njs_json_parse_ctx_t *ctx, njs_value_t *value,
                     continue;
                 }
 
-                p += 2;
+                utf_low = njs_json_unicode(p + 2);
 
-                utf_low = njs_json_unicode(p);
-                p += 4;
-
-                if (njs_fast_path(njs_surrogate_trailing(utf_low))) {
-                    utf = njs_surrogate_pair(utf, utf_low);
-
-                } else if (njs_surrogate_leading(utf_low)) {
-                    utf = NJS_UNICODE_REPLACEMENT;
-                    s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT);
-
-                } else {
-                    utf = utf_low;
+                if (njs_slow_path(!njs_surrogate_trailing(utf_low))) {
+                    /*
+                     * The next escape is left to the next iteration,
+                     * a leading surrogate there can start a new pair.
+                     */
                     s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT);
+                    continue;
                 }
+
+                p += 6;
+
+                utf = njs_surrogate_pair(utf, utf_low);
             }
 
             s = njs_utf8_encode(s, utf);
index 53c132d4ffd31f181ee295d82a8010481c30e29b..b4d42fa42714d28ac3b79b78d0ba4df6dbd5e53c 100644 (file)
@@ -8800,18 +8800,14 @@ next_char:
             if (njs_fast_path(njs_surrogate_trailing(cp))) {
                 cp = njs_surrogate_pair(cp_pair, cp);
 
-            } else if (njs_slow_path(njs_surrogate_leading(cp))) {
-                cp = NJS_UNICODE_REPLACEMENT;
-
-                dst = njs_utf8_encode(dst, (uint32_t) cp);
-
             } else {
                 dst = njs_utf8_encode(dst, NJS_UNICODE_REPLACEMENT);
             }
 
             cp_pair = 0;
+        }
 
-        } else if (njs_surrogate_any(cp)) {
+        if (njs_surrogate_any(cp)) {
             if (cp <= 0xdbff && (end - src) >= 2
                 && src[0] == '\\' && src[1] == 'u')
             {
@@ -8957,20 +8953,15 @@ njs_parser_escape_string_calc_length(njs_parser_t *parser,
             if (njs_fast_path(njs_surrogate_trailing(cp))) {
                 cp = njs_surrogate_pair(cp_pair, cp);
 
-            } else if (njs_slow_path(njs_surrogate_leading(cp))) {
-                cp = NJS_UNICODE_REPLACEMENT;
-
-                size += njs_utf8_size(cp);
-                length++;
-
             } else {
                 size += njs_utf8_size(NJS_UNICODE_REPLACEMENT);
                 length++;
             }
 
             cp_pair = 0;
+        }
 
-        } else if (njs_surrogate_any(cp)) {
+        if (njs_surrogate_any(cp)) {
             if (cp <= 0xdbff && (end - src) >= 2
                 && src[0] == '\\' && src[1] == 'u')
             {
index be382b3ef52103f9769eac5126489aa5e0f489bb..7f3721fefc837601c10cd121097df4493976e83e 100644 (file)
@@ -8738,6 +8738,12 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("'\\u{d800}\\u{dB00}'"),
       njs_str("��") },
 
+    { njs_str("'\\ud83d\\ud83d\\ude00'"),
+      njs_str("\xef\xbf\xbd\xf0\x9f\x98\x80") },
+
+    { njs_str("'\\u{d83d}\\u{d83d}\\u{de00}'.length"),
+      njs_str("2") },
+
     { njs_str("'\\u{d800}\\u{d7ff}'"),
       njs_str("�퟿") },
 
@@ -19794,6 +19800,12 @@ static njs_unit_test_t  njs_test[] =
     { njs_str("JSON.parse('\"\\\\uD800\\\\uDB00\"')"),
       njs_str("��") },
 
+    { njs_str("JSON.parse('\"\\\\uD83D\\\\uD83D\\\\uDE00\"')"),
+      njs_str("\xef\xbf\xbd\xf0\x9f\x98\x80") },
+
+    { njs_str("JSON.parse('\"\\\\uD83D\\\\uD83D\\\\uDE00\"').length"),
+      njs_str("2") },
+
     { njs_str("JSON.parse('\"\\\\ud800[\"')"),
       njs_str("�[") },