From: basavaraj-sm05 Date: Sun, 15 Nov 2026 14:32:00 +0000 (+0530) Subject: TextDecoder: skip BOM only at the start of a stream X-Git-Url: https://git.kaiwu.me/%7Bsubstring-before(@doc,%20'.xml')%7D.html?a=commitdiff_plain;h=c1fc7f03160ca0a710066a2054c983a7732f2db3;p=njs.git TextDecoder: skip BOM only at the start of a stream Previously, decode() checked for a UTF-8 BOM at the start of every input chunk. This dropped U+FEFF within a stream and preserved a leading BOM split between chunks. This affected both engines. Now BOM handling follows the first decoded code point of the stream, as required by Encoding Standard sections 7.1 and 7.2. Empty chunks and incomplete UTF-8 sequences do not mark the BOM as seen. The decoder state is reset at the start of a new stream, including after a failed final decode(). The BOM seen flag is updated only when decoded output is returned, while the UTF-8 context advances up to the error in fatal mode. The issue dates back to 0570392f (0.4.3) and was copied into QuickJS in 447d66d4 (0.8.10). --- diff --git a/src/njs_encoding.c b/src/njs_encoding.c index 6cd298ea..5135d02d 100644 --- a/src/njs_encoding.c +++ b/src/njs_encoding.c @@ -17,6 +17,8 @@ typedef struct { njs_encoding_t encoding; njs_bool_t fatal; njs_bool_t ignore_bom; + njs_bool_t bom_seen; + njs_bool_t stream; njs_unicode_decode_t ctx; } njs_encoding_decode_t; @@ -332,6 +334,8 @@ njs_text_decoder_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, } njs_utf8_decode_init(&data->ctx); + data->bom_seen = 0; + data->stream = 0; njs_set_data(&decoder->value, data, NJS_DATA_TAG_TEXT_DECODER); njs_set_object_value(retval, decoder); @@ -491,10 +495,10 @@ njs_text_decoder_decode(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, size_t size; ssize_t length; njs_int_t ret; - njs_bool_t stream; + njs_bool_t stream, bom_seen; njs_value_t *this, *value, *options; const u_char *start, *end; - njs_unicode_decode_t ctx; + njs_unicode_decode_t ctx, next; njs_encoding_decode_t *data; const njs_typed_array_t *array; const njs_array_buffer_t *buffer; @@ -560,17 +564,27 @@ njs_text_decoder_decode(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, data = njs_object_data(this); + if (!data->stream) { + njs_utf8_decode_init(&data->ctx); + data->bom_seen = 0; + } + + data->stream = stream; ctx = data->ctx; + bom_seen = data->bom_seen; /* Looking for BOM. */ - if (start != NULL && !data->ignore_bom) { - start += njs_utf8_bom(start, end); + if (!data->ignore_bom && !bom_seen) { + bom_seen = njs_utf8_bom(&ctx, &start, end); } - length = njs_utf8_stream_length(&ctx, start, end - start, !stream, + next = ctx; + + length = njs_utf8_stream_length(&next, start, end - start, !stream, data->fatal, &size); if (length == -1) { + data->ctx = next; njs_type_error(vm, "The encoded data was not valid"); return NJS_ERROR; } @@ -580,11 +594,10 @@ njs_text_decoder_decode(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, return NJS_ERROR; } - (void) njs_utf8_stream_encode(&data->ctx, start, end, dst, !stream, 0); + (void) njs_utf8_stream_encode(&ctx, start, end, dst, !stream, 0); - if (!stream) { - njs_utf8_decode_init(&data->ctx); - } + data->ctx = ctx; + data->bom_seen = bom_seen; return NJS_OK; } diff --git a/src/njs_utf8.h b/src/njs_utf8.h index 7386b4b7..87908bbf 100644 --- a/src/njs_utf8.h +++ b/src/njs_utf8.h @@ -149,18 +149,33 @@ njs_utf8_length(const u_char *p, size_t len) } -njs_inline size_t -njs_utf8_bom(const u_char *start, const u_char *end) +njs_inline njs_bool_t +njs_utf8_bom(njs_unicode_decode_t *ctx, const u_char **start, + const u_char *end) { - if (start + 3 > end) { + uint32_t cp; + const u_char *p; + njs_unicode_decode_t next; + + if (*start == end) { + return 0; + } + + next = *ctx; + p = *start; + + cp = njs_utf8_decode(&next, &p, end); + + if (cp == NJS_UNICODE_CONTINUE) { return 0; } - if (start[0] == 0xEF && start[1] == 0xBB && start[2] == 0xBF) { - return 3; + if (cp == NJS_UNICODE_BOM) { + *ctx = next; + *start = p; } - return 0; + return 1; } diff --git a/src/qjs.c b/src/qjs.c index 43930657..ccbf8865 100644 --- a/src/qjs.c +++ b/src/qjs.c @@ -34,6 +34,8 @@ typedef struct { qjs_encoding_t encoding; int fatal; int ignore_bom; + int bom_seen; + int stream; njs_unicode_decode_t ctx; } qjs_text_decoder_t; @@ -891,7 +893,7 @@ static JSValue qjs_text_decoder_decode(JSContext *cx, JSValueConst this_val, int argc, JSValueConst *argv) { - int stream; + int stream, bom_seen; size_t size; u_char *dst; JSValue ret; @@ -899,7 +901,7 @@ qjs_text_decoder_decode(JSContext *cx, JSValueConst this_val, int argc, njs_str_t data; const u_char *end; qjs_text_decoder_t *td; - njs_unicode_decode_t ctx; + njs_unicode_decode_t ctx, next; td = JS_GetOpaque(this_val, QJS_CORE_CLASS_ID_TEXT_DECODER); if (td == NULL) { @@ -923,17 +925,28 @@ qjs_text_decoder_decode(JSContext *cx, JSValueConst this_val, int argc, JS_FreeValue(cx, ret); } - ctx = td->ctx; end = data.start + data.length; - if (data.start != NULL && !td->ignore_bom) { - data.start += njs_utf8_bom(data.start, end); + if (!td->stream) { + njs_utf8_decode_init(&td->ctx); + td->bom_seen = 0; + } + + td->stream = stream; + ctx = td->ctx; + bom_seen = td->bom_seen; + + if (!td->ignore_bom && !bom_seen) { + bom_seen = njs_utf8_bom(&ctx, (const u_char **) &data.start, end); } - length = njs_utf8_stream_length(&ctx, data.start, end - data.start, !stream, - td->fatal, &size); + next = ctx; + + length = njs_utf8_stream_length(&next, data.start, end - data.start, + !stream, td->fatal, &size); if (length == -1) { + td->ctx = next; return JS_ThrowTypeError(cx, "The encoded data was not valid"); } @@ -943,15 +956,18 @@ qjs_text_decoder_decode(JSContext *cx, JSValueConst this_val, int argc, return JS_EXCEPTION; } - (void) njs_utf8_stream_encode(&td->ctx, data.start, end, dst, !stream, 0); + (void) njs_utf8_stream_encode(&ctx, data.start, end, dst, !stream, 0); ret = JS_NewStringLen(cx, (const char *) dst, size); js_free(cx, dst); - if (!stream) { - njs_utf8_decode_init(&td->ctx); + if (JS_IsException(ret)) { + return ret; } + td->ctx = ctx; + td->bom_seen = bom_seen; + return ret; } diff --git a/test/text_decoder.t.js b/test/text_decoder.t.js index eaf7e191..737adadc 100644 --- a/test/text_decoder.t.js +++ b/test/text_decoder.t.js @@ -58,6 +58,20 @@ let stream_tsuite = { expected: ['', '', '𠮷'] }, { chunks: [new Uint8Array([0xF0, 0xA0]), new Uint8Array([])], expected: ['', '�'] }, + // BOM is ignored only at the start of the stream + { chunks: [new Uint8Array([0x41]), + new Uint8Array([0xEF, 0xBB, 0xBF, 0x42])], + expected: ['A', '\uFEFFB'] }, + { chunks: [new Uint8Array([0xEF, 0xBB, 0xBF]), + new Uint8Array([0xEF, 0xBB, 0xBF, 0x42])], + expected: ['', '\uFEFFB'] }, + { chunks: [new Uint8Array([0xEF]), new Uint8Array([0xBB, 0xBF, 0x43])], + expected: ['', 'C'] }, + { chunks: [new Uint8Array([0xEF, 0xBB]), new Uint8Array([0xBF]), + new Uint8Array([0x44])], + expected: ['', '', 'D'] }, + { chunks: [new Uint8Array([0xEF, 0xBB]), new Uint8Array([0x45])], + expected: ['', '�E'] }, { chunks: [''], exception: 'TypeError: TypeError: not a TypedArray' }, ], @@ -105,6 +119,59 @@ let fatal_tsuite = { ], }; +let reuse_tsuite = { + name: "TextDecoder() reuse tests", + T: async (params) => { + let td = new TextDecoder('utf-8', params.opts || {fatal: true}); + let chunks = []; + + for (let i = 0; i < params.chunks.length; i++) { + try { + chunks.push(td.decode(new Uint8Array(params.chunks[i]), + {stream: params.stream[i]})); + } catch (e) { + if (!e.toString().startsWith('TypeError:')) { + throw e; + } + + chunks.push(null); + } + } + + if (!compareArray(chunks, params.expected)) { + throw Error(`unexpected output ${JSON.stringify(chunks)}`); + } + + return 'SUCCESS'; + }, + + tests: [ + { chunks: [[0xFF], [0xEF, 0xBB, 0xBF, 0x41]], + stream: [false, false], expected: [null, 'A'] }, + { chunks: [[0xFF], [0xEF, 0xBB, 0xBF, 0x41]], + stream: [true, false], expected: [null, 'A'] }, + { chunks: [[0x41, 0xFF], [0xEF, 0xBB, 0xBF, 0x41]], + stream: [true, false], expected: [null, 'A'] }, + { chunks: [[0xEF, 0xBB, 0xBF, 0xFF], [0xEF, 0xBB, 0xBF, 0x41]], + stream: [true, false], expected: [null, 'A'] }, + { chunks: [[0xEF], [0xBB, 0xBF, 0xFF], [0xEF, 0xBB, 0xBF, 0x41]], + stream: [true, true, false], expected: ['', null, 'A'] }, + { chunks: [[0x41], [0xFF], [0xEF, 0xBB, 0xBF, 0x42]], + stream: [true, false, false], expected: ['A', null, 'B'] }, + { chunks: [[0x41], [0xFF], [0xEF, 0xBB, 0xBF, 0x42]], + stream: [true, true, false], expected: ['A', null, '\uFEFFB'] }, + { chunks: [[0xEF, 0xBB], [], [0xEF, 0xBB, 0xBF, 0x41]], + stream: [true, false, false], expected: ['', null, 'A'] }, + { chunks: [[0xEF, 0xBB, 0xBF, 0x41], [0xEF, 0xBB, 0xBF, 0x42]], + stream: [false, false], expected: ['A', 'B'] }, + { chunks: [[], [0xEF], [], [0xBB], [0xBF], []], + stream: [true, true, true, true, true, false], + expected: ['', '', '', '', '', ''] }, + { chunks: [[0xEF], [0xBB], [0xBF]], opts: {ignoreBOM: true}, + stream: [true, true, false], expected: ['', '', '\uFEFF'] }, + ], +}; + let ignoreBOM_tsuite = { name: "TextDecoder() ignoreBOM tests", T: async (params) => { @@ -166,6 +233,7 @@ let detached_tsuite = { run([ stream_tsuite, fatal_tsuite, + reuse_tsuite, ignoreBOM_tsuite, detached_tsuite, ])