njs_encoding_t encoding;
njs_bool_t fatal;
njs_bool_t ignore_bom;
+ njs_bool_t bom_seen;
+ njs_bool_t stream;
njs_unicode_decode_t ctx;
} njs_encoding_decode_t;
}
njs_utf8_decode_init(&data->ctx);
+ data->bom_seen = 0;
+ data->stream = 0;
njs_set_data(&decoder->value, data, NJS_DATA_TAG_TEXT_DECODER);
njs_set_object_value(retval, decoder);
size_t size;
ssize_t length;
njs_int_t ret;
- njs_bool_t stream;
+ njs_bool_t stream, bom_seen;
njs_value_t *this, *value, *options;
const u_char *start, *end;
- njs_unicode_decode_t ctx;
+ njs_unicode_decode_t ctx, next;
njs_encoding_decode_t *data;
const njs_typed_array_t *array;
const njs_array_buffer_t *buffer;
data = njs_object_data(this);
+ if (!data->stream) {
+ njs_utf8_decode_init(&data->ctx);
+ data->bom_seen = 0;
+ }
+
+ data->stream = stream;
ctx = data->ctx;
+ bom_seen = data->bom_seen;
/* Looking for BOM. */
- if (start != NULL && !data->ignore_bom) {
- start += njs_utf8_bom(start, end);
+ if (!data->ignore_bom && !bom_seen) {
+ bom_seen = njs_utf8_bom(&ctx, &start, end);
}
- length = njs_utf8_stream_length(&ctx, start, end - start, !stream,
+ next = ctx;
+
+ length = njs_utf8_stream_length(&next, start, end - start, !stream,
data->fatal, &size);
if (length == -1) {
+ data->ctx = next;
njs_type_error(vm, "The encoded data was not valid");
return NJS_ERROR;
}
return NJS_ERROR;
}
- (void) njs_utf8_stream_encode(&data->ctx, start, end, dst, !stream, 0);
+ (void) njs_utf8_stream_encode(&ctx, start, end, dst, !stream, 0);
- if (!stream) {
- njs_utf8_decode_init(&data->ctx);
- }
+ data->ctx = ctx;
+ data->bom_seen = bom_seen;
return NJS_OK;
}
qjs_encoding_t encoding;
int fatal;
int ignore_bom;
+ int bom_seen;
+ int stream;
njs_unicode_decode_t ctx;
} qjs_text_decoder_t;
qjs_text_decoder_decode(JSContext *cx, JSValueConst this_val, int argc,
JSValueConst *argv)
{
- int stream;
+ int stream, bom_seen;
size_t size;
u_char *dst;
JSValue ret;
njs_str_t data;
const u_char *end;
qjs_text_decoder_t *td;
- njs_unicode_decode_t ctx;
+ njs_unicode_decode_t ctx, next;
td = JS_GetOpaque(this_val, QJS_CORE_CLASS_ID_TEXT_DECODER);
if (td == NULL) {
JS_FreeValue(cx, ret);
}
- ctx = td->ctx;
end = data.start + data.length;
- if (data.start != NULL && !td->ignore_bom) {
- data.start += njs_utf8_bom(data.start, end);
+ if (!td->stream) {
+ njs_utf8_decode_init(&td->ctx);
+ td->bom_seen = 0;
+ }
+
+ td->stream = stream;
+ ctx = td->ctx;
+ bom_seen = td->bom_seen;
+
+ if (!td->ignore_bom && !bom_seen) {
+ bom_seen = njs_utf8_bom(&ctx, (const u_char **) &data.start, end);
}
- length = njs_utf8_stream_length(&ctx, data.start, end - data.start, !stream,
- td->fatal, &size);
+ next = ctx;
+
+ length = njs_utf8_stream_length(&next, data.start, end - data.start,
+ !stream, td->fatal, &size);
if (length == -1) {
+ td->ctx = next;
return JS_ThrowTypeError(cx, "The encoded data was not valid");
}
return JS_EXCEPTION;
}
- (void) njs_utf8_stream_encode(&td->ctx, data.start, end, dst, !stream, 0);
+ (void) njs_utf8_stream_encode(&ctx, data.start, end, dst, !stream, 0);
ret = JS_NewStringLen(cx, (const char *) dst, size);
js_free(cx, dst);
- if (!stream) {
- njs_utf8_decode_init(&td->ctx);
+ if (JS_IsException(ret)) {
+ return ret;
}
+ td->ctx = ctx;
+ td->bom_seen = bom_seen;
+
return ret;
}
expected: ['', '', '𠮷'] },
{ chunks: [new Uint8Array([0xF0, 0xA0]), new Uint8Array([])],
expected: ['', '�'] },
+ // BOM is ignored only at the start of the stream
+ { chunks: [new Uint8Array([0x41]),
+ new Uint8Array([0xEF, 0xBB, 0xBF, 0x42])],
+ expected: ['A', '\uFEFFB'] },
+ { chunks: [new Uint8Array([0xEF, 0xBB, 0xBF]),
+ new Uint8Array([0xEF, 0xBB, 0xBF, 0x42])],
+ expected: ['', '\uFEFFB'] },
+ { chunks: [new Uint8Array([0xEF]), new Uint8Array([0xBB, 0xBF, 0x43])],
+ expected: ['', 'C'] },
+ { chunks: [new Uint8Array([0xEF, 0xBB]), new Uint8Array([0xBF]),
+ new Uint8Array([0x44])],
+ expected: ['', '', 'D'] },
+ { chunks: [new Uint8Array([0xEF, 0xBB]), new Uint8Array([0x45])],
+ expected: ['', '�E'] },
{ chunks: [''],
exception: 'TypeError: TypeError: not a TypedArray' },
],
],
};
+let reuse_tsuite = {
+ name: "TextDecoder() reuse tests",
+ T: async (params) => {
+ let td = new TextDecoder('utf-8', params.opts || {fatal: true});
+ let chunks = [];
+
+ for (let i = 0; i < params.chunks.length; i++) {
+ try {
+ chunks.push(td.decode(new Uint8Array(params.chunks[i]),
+ {stream: params.stream[i]}));
+ } catch (e) {
+ if (!e.toString().startsWith('TypeError:')) {
+ throw e;
+ }
+
+ chunks.push(null);
+ }
+ }
+
+ if (!compareArray(chunks, params.expected)) {
+ throw Error(`unexpected output ${JSON.stringify(chunks)}`);
+ }
+
+ return 'SUCCESS';
+ },
+
+ tests: [
+ { chunks: [[0xFF], [0xEF, 0xBB, 0xBF, 0x41]],
+ stream: [false, false], expected: [null, 'A'] },
+ { chunks: [[0xFF], [0xEF, 0xBB, 0xBF, 0x41]],
+ stream: [true, false], expected: [null, 'A'] },
+ { chunks: [[0x41, 0xFF], [0xEF, 0xBB, 0xBF, 0x41]],
+ stream: [true, false], expected: [null, 'A'] },
+ { chunks: [[0xEF, 0xBB, 0xBF, 0xFF], [0xEF, 0xBB, 0xBF, 0x41]],
+ stream: [true, false], expected: [null, 'A'] },
+ { chunks: [[0xEF], [0xBB, 0xBF, 0xFF], [0xEF, 0xBB, 0xBF, 0x41]],
+ stream: [true, true, false], expected: ['', null, 'A'] },
+ { chunks: [[0x41], [0xFF], [0xEF, 0xBB, 0xBF, 0x42]],
+ stream: [true, false, false], expected: ['A', null, 'B'] },
+ { chunks: [[0x41], [0xFF], [0xEF, 0xBB, 0xBF, 0x42]],
+ stream: [true, true, false], expected: ['A', null, '\uFEFFB'] },
+ { chunks: [[0xEF, 0xBB], [], [0xEF, 0xBB, 0xBF, 0x41]],
+ stream: [true, false, false], expected: ['', null, 'A'] },
+ { chunks: [[0xEF, 0xBB, 0xBF, 0x41], [0xEF, 0xBB, 0xBF, 0x42]],
+ stream: [false, false], expected: ['A', 'B'] },
+ { chunks: [[], [0xEF], [], [0xBB], [0xBF], []],
+ stream: [true, true, true, true, true, false],
+ expected: ['', '', '', '', '', ''] },
+ { chunks: [[0xEF], [0xBB], [0xBF]], opts: {ignoreBOM: true},
+ stream: [true, true, false], expected: ['', '', '\uFEFF'] },
+ ],
+};
+
let ignoreBOM_tsuite = {
name: "TextDecoder() ignoreBOM tests",
T: async (params) => {
run([
stream_tsuite,
fatal_tsuite,
+ reuse_tsuite,
ignoreBOM_tsuite,
detached_tsuite,
])