From: Dmitry Volyntsev Date: Sat, 14 Nov 2026 02:32:10 +0000 (-0800) Subject: Core: fix buffering of 0xFF in the UTF-16 decoder X-Git-Url: https://git.kaiwu.me/ngx_http_proxy_module.html?a=commitdiff_plain;h=91ecb9f01510861e77d982d89afcb7daa3e59345;p=njs.git Core: fix buffering of 0xFF in the UTF-16 decoder Previously, the UTF-16 decoder stored a buffered byte plus one in the eight-bit ctx->upper field, using zero to mean no buffered byte. A byte of 0xFF overflowed to zero and was lost. This corrupted code units split across buffers and units reread after an unpaired leading surrogate. Now ctx->need indicates whether a byte is buffered, and ctx->upper stores the byte unchanged. All byte values can be preserved without changing the decode context layout or its streaming contract. The issue was introduced in 29271d0a (0.4.3). --- diff --git a/src/njs_utf16.c b/src/njs_utf16.c index b47fbf50..a3085e52 100644 --- a/src/njs_utf16.c +++ b/src/njs_utf16.c @@ -54,9 +54,9 @@ njs_utf16_decode(njs_unicode_decode_t *ctx, const u_char **start, uint32_t unit; unsigned lead; - if (ctx->upper != 0x00) { - lead = ctx->upper - 0x01; - ctx->upper = 0x00; + if (ctx->need != 0x00) { + lead = ctx->upper; + ctx->need = 0x00; goto lead_state; } @@ -66,7 +66,8 @@ pair_state: lead = *(*start)++; if (*start >= end) { - ctx->upper = lead + 0x01; + ctx->upper = lead; + ctx->need = 0x01; return NJS_UNICODE_CONTINUE; } @@ -89,7 +90,8 @@ lead_state: (*start)--; - ctx->upper = lead + 0x01; + ctx->upper = lead; + ctx->need = 0x01; ctx->codepoint = 0x00; return NJS_UNICODE_ERROR; diff --git a/src/njs_utf16.h b/src/njs_utf16.h index b5675586..d05a8175 100644 --- a/src/njs_utf16.h +++ b/src/njs_utf16.h @@ -17,6 +17,7 @@ NJS_EXPORT uint32_t njs_utf16_decode(njs_unicode_decode_t *ctx, njs_inline void njs_utf16_decode_init(njs_unicode_decode_t *ctx) { + ctx->need = 0x00; ctx->upper = 0x00; ctx->codepoint = 0x00; } diff --git a/src/test/unicode_unit_test.c b/src/test/unicode_unit_test.c index 35416ea1..6d8540ec 100644 --- a/src/test/unicode_unit_test.c +++ b/src/test/unicode_unit_test.c @@ -211,7 +211,7 @@ utf16_unit_test() { int8_t length, length_to; u_char *start, *end, *end_to; - uint32_t cp, i; + uint32_t cp, i, j; njs_unicode_decode_t ctx; u_char buf[8], to[4]; @@ -263,6 +263,43 @@ utf16_unit_test() njs_printf("utf16 test decode-encode failed\n"); return NJS_ERROR; } + + njs_utf16_decode_init(&ctx); + start = buf; + + for (j = 0; j < (uint32_t) length; j++) { + cp = njs_utf16_decode(&ctx, (const u_char **) &start, start + 1); + + if (cp != (j + 1 == (uint32_t) length ? i : NJS_UNICODE_CONTINUE)) { + njs_printf("utf16 test byte decoding failed for %05uXD\n", i); + return NJS_ERROR; + } + } + + start = to; + (void) njs_utf16_encode(0xD800, &start, end_to); + + njs_utf16_decode_init(&ctx); + start = to; + + cp = njs_utf16_decode(&ctx, (const u_char **) &start, to + 2); + if (cp != NJS_UNICODE_CONTINUE) { + njs_printf("utf16 test pending surrogate failed\n"); + return NJS_ERROR; + } + + start = buf; + cp = njs_utf16_decode(&ctx, (const u_char **) &start, buf + length); + if (cp != NJS_UNICODE_ERROR || start != buf + 1) { + njs_printf("utf16 test unpaired surrogate failed for %05uXD\n", i); + return NJS_ERROR; + } + + cp = njs_utf16_decode(&ctx, (const u_char **) &start, buf + length); + if (cp != i || start != buf + length) { + njs_printf("utf16 test rereading code point failed for %05uXD\n", i); + return NJS_ERROR; + } } /* Surrogate pair. */