]> git.kaiwu.me - njs.git/commitdiff
TextDecoder: skip BOM only at the start of a stream
authorbasavaraj-sm05 <basavaraj@digiscrypt.com>
Sun, 15 Nov 2026 14:32:00 +0000 (20:02 +0530)
committerDmitry Volyntsev <xeioexception@gmail.com>
Mon, 16 Nov 2026 17:10:26 +0000 (09:10 -0800)
Previously, decode() checked for a UTF-8 BOM at the start of every
input chunk.  This dropped U+FEFF within a stream and preserved a
leading BOM split between chunks.  This affected both engines.

Now BOM handling follows the first decoded code point of the stream,
as required by Encoding Standard sections 7.1 and 7.2.  Empty chunks
and incomplete UTF-8 sequences do not mark the BOM as seen.

The decoder state is reset at the start of a new stream, including
after a failed final decode().  The BOM seen flag is updated only
when decoded output is returned, while the UTF-8 context advances
up to the error in fatal mode.

The issue dates back to 0570392f (0.4.3) and was copied into QuickJS
in 447d66d4 (0.8.10).

src/njs_encoding.c
src/njs_utf8.h
src/qjs.c
test/text_decoder.t.js

index 6cd298ea554d2239b782bdb9fa8cb93a0f84a7a2..5135d02d5a1590e8f46e08d27373aadcae495bfa 100644 (file)
@@ -17,6 +17,8 @@ typedef struct {
     njs_encoding_t        encoding;
     njs_bool_t            fatal;
     njs_bool_t            ignore_bom;
+    njs_bool_t            bom_seen;
+    njs_bool_t            stream;
 
     njs_unicode_decode_t  ctx;
 } njs_encoding_decode_t;
@@ -332,6 +334,8 @@ njs_text_decoder_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     }
 
     njs_utf8_decode_init(&data->ctx);
+    data->bom_seen = 0;
+    data->stream = 0;
 
     njs_set_data(&decoder->value, data, NJS_DATA_TAG_TEXT_DECODER);
     njs_set_object_value(retval, decoder);
@@ -491,10 +495,10 @@ njs_text_decoder_decode(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     size_t                    size;
     ssize_t                   length;
     njs_int_t                 ret;
-    njs_bool_t                stream;
+    njs_bool_t                stream, bom_seen;
     njs_value_t               *this, *value, *options;
     const u_char              *start, *end;
-    njs_unicode_decode_t      ctx;
+    njs_unicode_decode_t      ctx, next;
     njs_encoding_decode_t     *data;
     const njs_typed_array_t   *array;
     const njs_array_buffer_t  *buffer;
@@ -560,17 +564,27 @@ njs_text_decoder_decode(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
 
     data = njs_object_data(this);
 
+    if (!data->stream) {
+        njs_utf8_decode_init(&data->ctx);
+        data->bom_seen = 0;
+    }
+
+    data->stream = stream;
     ctx = data->ctx;
+    bom_seen = data->bom_seen;
 
     /* Looking for BOM. */
 
-    if (start != NULL && !data->ignore_bom) {
-        start += njs_utf8_bom(start, end);
+    if (!data->ignore_bom && !bom_seen) {
+        bom_seen = njs_utf8_bom(&ctx, &start, end);
     }
 
-    length = njs_utf8_stream_length(&ctx, start, end - start, !stream,
+    next = ctx;
+
+    length = njs_utf8_stream_length(&next, start, end - start, !stream,
                                     data->fatal, &size);
     if (length == -1) {
+        data->ctx = next;
         njs_type_error(vm, "The encoded data was not valid");
         return NJS_ERROR;
     }
@@ -580,11 +594,10 @@ njs_text_decoder_decode(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
         return NJS_ERROR;
     }
 
-    (void) njs_utf8_stream_encode(&data->ctx, start, end, dst, !stream, 0);
+    (void) njs_utf8_stream_encode(&ctx, start, end, dst, !stream, 0);
 
-    if (!stream) {
-        njs_utf8_decode_init(&data->ctx);
-    }
+    data->ctx = ctx;
+    data->bom_seen = bom_seen;
 
     return NJS_OK;
 }
index 7386b4b7be3439f37cf64d7d437e85cb0a5fa7b4..87908bbf950adaf94be5882ae299329118eba2f6 100644 (file)
@@ -149,18 +149,33 @@ njs_utf8_length(const u_char *p, size_t len)
 }
 
 
-njs_inline size_t
-njs_utf8_bom(const u_char *start, const u_char *end)
+njs_inline njs_bool_t
+njs_utf8_bom(njs_unicode_decode_t *ctx, const u_char **start,
+    const u_char *end)
 {
-    if (start + 3 > end) {
+    uint32_t              cp;
+    const u_char          *p;
+    njs_unicode_decode_t  next;
+
+    if (*start == end) {
+        return 0;
+    }
+
+    next = *ctx;
+    p = *start;
+
+    cp = njs_utf8_decode(&next, &p, end);
+
+    if (cp == NJS_UNICODE_CONTINUE) {
         return 0;
     }
 
-    if (start[0] == 0xEF && start[1] == 0xBB && start[2] == 0xBF) {
-        return 3;
+    if (cp == NJS_UNICODE_BOM) {
+        *ctx = next;
+        *start = p;
     }
 
-    return 0;
+    return 1;
 }
 
 
index 4393065737dc37c6c6b65e80b1d81b9660600eeb..ccbf886567ac84bda891dcf3165609df714c384b 100644 (file)
--- a/src/qjs.c
+++ b/src/qjs.c
@@ -34,6 +34,8 @@ typedef struct {
     qjs_encoding_t        encoding;
     int                   fatal;
     int                   ignore_bom;
+    int                   bom_seen;
+    int                   stream;
 
     njs_unicode_decode_t  ctx;
 } qjs_text_decoder_t;
@@ -891,7 +893,7 @@ static JSValue
 qjs_text_decoder_decode(JSContext *cx, JSValueConst this_val, int argc,
     JSValueConst *argv)
 {
-    int                   stream;
+    int                   stream, bom_seen;
     size_t                size;
     u_char                *dst;
     JSValue               ret;
@@ -899,7 +901,7 @@ qjs_text_decoder_decode(JSContext *cx, JSValueConst this_val, int argc,
     njs_str_t             data;
     const u_char          *end;
     qjs_text_decoder_t    *td;
-    njs_unicode_decode_t  ctx;
+    njs_unicode_decode_t  ctx, next;
 
     td = JS_GetOpaque(this_val, QJS_CORE_CLASS_ID_TEXT_DECODER);
     if (td == NULL) {
@@ -923,17 +925,28 @@ qjs_text_decoder_decode(JSContext *cx, JSValueConst this_val, int argc,
         JS_FreeValue(cx, ret);
     }
 
-    ctx = td->ctx;
     end = data.start + data.length;
 
-    if (data.start != NULL && !td->ignore_bom) {
-        data.start += njs_utf8_bom(data.start, end);
+    if (!td->stream) {
+        njs_utf8_decode_init(&td->ctx);
+        td->bom_seen = 0;
+    }
+
+    td->stream = stream;
+    ctx = td->ctx;
+    bom_seen = td->bom_seen;
+
+    if (!td->ignore_bom && !bom_seen) {
+        bom_seen = njs_utf8_bom(&ctx, (const u_char **) &data.start, end);
     }
 
-    length = njs_utf8_stream_length(&ctx, data.start, end - data.start, !stream,
-                                    td->fatal, &size);
+    next = ctx;
+
+    length = njs_utf8_stream_length(&next, data.start, end - data.start,
+                                    !stream, td->fatal, &size);
 
     if (length == -1) {
+        td->ctx = next;
         return JS_ThrowTypeError(cx, "The encoded data was not valid");
     }
 
@@ -943,15 +956,18 @@ qjs_text_decoder_decode(JSContext *cx, JSValueConst this_val, int argc,
         return JS_EXCEPTION;
     }
 
-    (void) njs_utf8_stream_encode(&td->ctx, data.start, end, dst, !stream, 0);
+    (void) njs_utf8_stream_encode(&ctx, data.start, end, dst, !stream, 0);
 
     ret = JS_NewStringLen(cx, (const char *) dst, size);
     js_free(cx, dst);
 
-    if (!stream) {
-        njs_utf8_decode_init(&td->ctx);
+    if (JS_IsException(ret)) {
+        return ret;
     }
 
+    td->ctx = ctx;
+    td->bom_seen = bom_seen;
+
     return ret;
 }
 
index eaf7e191ed09a4dad47f2da7a5592c1efd80d50f..737adadcf2ea21871491dddbd7483a3bf1c199e6 100644 (file)
@@ -58,6 +58,20 @@ let stream_tsuite = {
           expected: ['', '', '𠮷'] },
         { chunks: [new Uint8Array([0xF0, 0xA0]), new Uint8Array([])],
           expected: ['', '�'] },
+        // BOM is ignored only at the start of the stream
+        { chunks: [new Uint8Array([0x41]),
+                   new Uint8Array([0xEF, 0xBB, 0xBF, 0x42])],
+          expected: ['A', '\uFEFFB'] },
+        { chunks: [new Uint8Array([0xEF, 0xBB, 0xBF]),
+                   new Uint8Array([0xEF, 0xBB, 0xBF, 0x42])],
+          expected: ['', '\uFEFFB'] },
+        { chunks: [new Uint8Array([0xEF]), new Uint8Array([0xBB, 0xBF, 0x43])],
+          expected: ['', 'C'] },
+        { chunks: [new Uint8Array([0xEF, 0xBB]), new Uint8Array([0xBF]),
+                   new Uint8Array([0x44])],
+          expected: ['', '', 'D'] },
+        { chunks: [new Uint8Array([0xEF, 0xBB]), new Uint8Array([0x45])],
+          expected: ['', '�E'] },
         { chunks: [''],
           exception: 'TypeError: TypeError: not a TypedArray' },
     ],
@@ -105,6 +119,59 @@ let fatal_tsuite = {
     ],
 };
 
+let reuse_tsuite = {
+    name: "TextDecoder() reuse tests",
+    T: async (params) => {
+        let td = new TextDecoder('utf-8', params.opts || {fatal: true});
+        let chunks = [];
+
+        for (let i = 0; i < params.chunks.length; i++) {
+            try {
+                chunks.push(td.decode(new Uint8Array(params.chunks[i]),
+                                      {stream: params.stream[i]}));
+            } catch (e) {
+                if (!e.toString().startsWith('TypeError:')) {
+                    throw e;
+                }
+
+                chunks.push(null);
+            }
+        }
+
+        if (!compareArray(chunks, params.expected)) {
+            throw Error(`unexpected output ${JSON.stringify(chunks)}`);
+        }
+
+        return 'SUCCESS';
+    },
+
+    tests: [
+        { chunks: [[0xFF], [0xEF, 0xBB, 0xBF, 0x41]],
+          stream: [false, false], expected: [null, 'A'] },
+        { chunks: [[0xFF], [0xEF, 0xBB, 0xBF, 0x41]],
+          stream: [true, false], expected: [null, 'A'] },
+        { chunks: [[0x41, 0xFF], [0xEF, 0xBB, 0xBF, 0x41]],
+          stream: [true, false], expected: [null, 'A'] },
+        { chunks: [[0xEF, 0xBB, 0xBF, 0xFF], [0xEF, 0xBB, 0xBF, 0x41]],
+          stream: [true, false], expected: [null, 'A'] },
+        { chunks: [[0xEF], [0xBB, 0xBF, 0xFF], [0xEF, 0xBB, 0xBF, 0x41]],
+          stream: [true, true, false], expected: ['', null, 'A'] },
+        { chunks: [[0x41], [0xFF], [0xEF, 0xBB, 0xBF, 0x42]],
+          stream: [true, false, false], expected: ['A', null, 'B'] },
+        { chunks: [[0x41], [0xFF], [0xEF, 0xBB, 0xBF, 0x42]],
+          stream: [true, true, false], expected: ['A', null, '\uFEFFB'] },
+        { chunks: [[0xEF, 0xBB], [], [0xEF, 0xBB, 0xBF, 0x41]],
+          stream: [true, false, false], expected: ['', null, 'A'] },
+        { chunks: [[0xEF, 0xBB, 0xBF, 0x41], [0xEF, 0xBB, 0xBF, 0x42]],
+          stream: [false, false], expected: ['A', 'B'] },
+        { chunks: [[], [0xEF], [], [0xBB], [0xBF], []],
+          stream: [true, true, true, true, true, false],
+          expected: ['', '', '', '', '', ''] },
+        { chunks: [[0xEF], [0xBB], [0xBF]], opts: {ignoreBOM: true},
+          stream: [true, true, false], expected: ['', '', '\uFEFF'] },
+    ],
+};
+
 let ignoreBOM_tsuite = {
     name: "TextDecoder() ignoreBOM tests",
     T: async (params) => {
@@ -166,6 +233,7 @@ let detached_tsuite = {
 run([
     stream_tsuite,
     fatal_tsuite,
+    reuse_tsuite,
     ignoreBOM_tsuite,
     detached_tsuite,
 ])