diff options
| -rw-r--r-- | ext/node/polyfills/string_decoder.ts | 593 | ||||
| -rw-r--r-- | tests/node_compat/config.jsonc | 1 | ||||
| -rw-r--r-- | tests/node_compat/test.ts | 1 | ||||
| -rw-r--r-- | tests/node_compat/test/parallel/test-string-decoder.js | 292 | ||||
| -rw-r--r-- | tools/node_compat/TODO.md | 1 |
5 files changed, 625 insertions, 263 deletions
diff --git a/ext/node/polyfills/string_decoder.ts b/ext/node/polyfills/string_decoder.ts index 507a994bb..ef83b6fc9 100644 --- a/ext/node/polyfills/string_decoder.ts +++ b/ext/node/polyfills/string_decoder.ts @@ -23,23 +23,38 @@ // TODO(petamoriken): enable prefer-primordials for node polyfills // deno-lint-ignore-file prefer-primordials -import { Buffer } from "node:buffer"; +// Logic and comments translated pretty much one-to-one from node's impl +// (https://github.com/nodejs/node/blob/ba06c5c509956dc413f91b755c1c93798bb700d4/src/string_decoder.cc) + +import { Buffer, constants } from "node:buffer"; +import { normalizeEncoding as castEncoding } from "ext:deno_node/_utils.ts"; import { - normalizeEncoding as castEncoding, - notImplemented, -} from "ext:deno_node/_utils.ts"; - -enum NotImplemented { - "ascii", - "latin1", - "utf16le", -} + ERR_INVALID_ARG_TYPE, + ERR_INVALID_THIS, + ERR_UNKNOWN_ENCODING, + NodeError, +} from "ext:deno_node/internal/errors.ts"; + +import { primordials } from "ext:core/mod.js"; +const { + ArrayBufferIsView, + ObjectDefineProperties, +} = primordials; + +const { MAX_STRING_LENGTH } = constants; + +// to cast from string to `BufferEncoding`, which doesn't seem nameable from here +// deno-lint-ignore no-explicit-any +type Any = any; function normalizeEncoding(enc?: string): string { const encoding = castEncoding(enc ?? null); - if (encoding && encoding in NotImplemented) notImplemented(encoding); - if (!encoding && typeof enc === "string" && enc.toLowerCase() !== "raw") { - throw new Error(`Unknown encoding: ${enc}`); + if (!encoding) { + if (typeof enc !== "string" || enc.toLowerCase() !== "raw") { + throw new ERR_UNKNOWN_ENCODING( + enc as Any, + ); + } } return String(encoding); } @@ -49,295 +64,349 @@ function normalizeEncoding(enc?: string): string { */ function isBufferType(buf: Buffer) { - return buf instanceof ArrayBuffer && buf.BYTES_PER_ELEMENT; + return buf instanceof Buffer && buf.BYTES_PER_ELEMENT; } -/* - * Checks the type of a UTF-8 byte, whether it's ASCII, a leading byte, or a - * continuation byte. If an invalid byte is detected, -2 is returned. - */ -function utf8CheckByte(byte: number): number { - if (byte <= 0x7f) return 0; - else if (byte >> 5 === 0x06) return 2; - else if (byte >> 4 === 0x0e) return 3; - else if (byte >> 3 === 0x1e) return 4; - return byte >> 6 === 0x02 ? -1 : -2; -} - -/* - * Checks at most 3 bytes at the end of a Buffer in order to detect an - * incomplete multi-byte UTF-8 character. The total number of bytes (2, 3, or 4) - * needed to complete the UTF-8 character (if applicable) are returned. - */ -function utf8CheckIncomplete( - self: StringDecoderBase, - buf: Buffer, - i: number, -): number { - let j = buf.length - 1; - if (j < i) return 0; - let nb = utf8CheckByte(buf[j]); - if (nb >= 0) { - if (nb > 0) self.lastNeed = nb - 1; - return nb; - } - if (--j < i || nb === -2) return 0; - nb = utf8CheckByte(buf[j]); - if (nb >= 0) { - if (nb > 0) self.lastNeed = nb - 2; - return nb; +function normalizeBuffer(buf: Buffer) { + if (!ArrayBufferIsView(buf)) { + throw new ERR_INVALID_ARG_TYPE( + "buf", + ["Buffer", "TypedArray", "DataView"], + buf, + ); } - if (--j < i || nb === -2) return 0; - nb = utf8CheckByte(buf[j]); - if (nb >= 0) { - if (nb > 0) { - if (nb === 2) nb = 0; - else self.lastNeed = nb - 3; - } - return nb; + if (isBufferType(buf)) { + return buf; + } else { + return Buffer.from( + buf.buffer, + ); } - return 0; } -/* - * Validates as many continuation bytes for a multi-byte UTF-8 character as - * needed or are available. If we see a non-continuation byte where we expect - * one, we "replace" the validated continuation bytes we've seen so far with - * a single UTF-8 replacement character ('\ufffd'), to match v8's UTF-8 decoding - * behavior. The continuation byte check is included three times in the case - * where all of the continuation bytes for a character exist in the same buffer. - * It is also done this way as a slight performance increase instead of using a - * loop. - */ -function utf8CheckExtraBytes( - self: StringDecoderBase, +function bufferToString( buf: Buffer, -): string | undefined { - if ((buf[0] & 0xc0) !== 0x80) { - self.lastNeed = 0; - return "\ufffd"; + encoding?: string, + start?: number, + end?: number, +): string { + const len = (end ?? buf.length) - (start ?? 0); + if (len > MAX_STRING_LENGTH) { + throw new NodeError("ERR_STRING_TOO_LONG", "string exceeds maximum length"); } - if (self.lastNeed > 1 && buf.length > 1) { - if ((buf[1] & 0xc0) !== 0x80) { - self.lastNeed = 1; - return "\ufffd"; - } - if (self.lastNeed > 2 && buf.length > 2) { - if ((buf[2] & 0xc0) !== 0x80) { - self.lastNeed = 2; - return "\ufffd"; + return buf.toString(encoding as Any, start, end); +} + +// the heart of the logic, decodes a buffer, storing +// incomplete characters in a buffer if applicable +function decode(this: StringDecoder, buf: Buffer) { + const enc = this.enc; + + let bufIdx = 0; + let bufEnd = buf.length; + + let prepend = ""; + let rest = ""; + + if ( + enc === Encoding.Utf8 || enc === Encoding.Utf16 || enc === Encoding.Base64 + ) { + // check if we need to finish an incomplete char from the last chunk + // written. If we do, we copy the bytes into our `lastChar` buffer + // and prepend the completed char to the result of decoding the rest of the buffer + if (this[kMissingBytes] > 0) { + if (enc === Encoding.Utf8) { + // Edge case for incomplete character at a chunk boundary + // (see https://github.com/nodejs/node/blob/73025c4dec042e344eeea7912ed39f7b7c4a3991/src/string_decoder.cc#L74) + for ( + let i = 0; + i < buf.length - bufIdx && i < this[kMissingBytes]; + i++ + ) { + if ((buf[i] & 0xC0) !== 0x80) { + // We expected a continuation byte, but got something else. + // Stop trying to decode the incomplete char, and assume + // the byte we got starts a new char. + this[kMissingBytes] = 0; + buf.copy(this.lastChar, this[kBufferedBytes], bufIdx, bufIdx + i); + this[kBufferedBytes] += i; + bufIdx += i; + break; + } + } + } + + const bytesToCopy = Math.min(buf.length - bufIdx, this[kMissingBytes]); + buf.copy( + this.lastChar, + this[kBufferedBytes], + bufIdx, + bufIdx + bytesToCopy, + ); + + bufIdx += bytesToCopy; + + this[kBufferedBytes] += bytesToCopy; + this[kMissingBytes] -= bytesToCopy; + + if (this[kMissingBytes] === 0) { + // we have all the bytes, complete the char + prepend = bufferToString( + this.lastChar, + this.encoding, + 0, + this[kBufferedBytes], + ); + // reset the char buffer + this[kBufferedBytes] = 0; } } - } -} -/* - * Attempts to complete a multi-byte UTF-8 character using bytes from a Buffer. - */ -function utf8FillLastComplete( - this: StringDecoderBase, - buf: Buffer, -): string | undefined { - const p = this.lastTotal - this.lastNeed; - const r = utf8CheckExtraBytes(this, buf); - if (r !== undefined) return r; - if (this.lastNeed <= buf.length) { - buf.copy(this.lastChar, p, 0, this.lastNeed); - return this.lastChar.toString(this.encoding, 0, this.lastTotal); - } - buf.copy(this.lastChar, p, 0, buf.length); - this.lastNeed -= buf.length; -} + if (buf.length - bufIdx === 0) { + // we advanced the bufIdx, so we may have completed the + // incomplete char + rest = prepend.length > 0 ? prepend : ""; + prepend = ""; + } else { + // no characters left to finish -/* - * Attempts to complete a partial non-UTF-8 character using bytes from a Buffer - */ -function utf8FillLastIncomplete( - this: StringDecoderBase, - buf: Buffer, -): string | undefined { - if (this.lastNeed <= buf.length) { - buf.copy(this.lastChar, this.lastTotal - this.lastNeed, 0, this.lastNeed); - return this.lastChar.toString(this.encoding, 0, this.lastTotal); - } - buf.copy(this.lastChar, this.lastTotal - this.lastNeed, 0, buf.length); - this.lastNeed -= buf.length; -} + // check if the end of the buffer has an incomplete + // character, if so we write it into our `lastChar` buffer and + // truncate buf + if (enc === Encoding.Utf8 && (buf[buf.length - 1] & 0x80)) { + for (let i = buf.length - 1;; i--) { + this[kBufferedBytes] += 1; + if ((buf[i] & 0xC0) === 0x80) { + // Doesn't start a character (i.e. it's a trailing byte) + if (this[kBufferedBytes] >= 4 || i === 0) { + // invalid utf8, we'll just pass it to the underlying decoder + this[kBufferedBytes] = 0; + break; + } + } else { + // First byte of a UTF-8 char, check + // to see how long it should be + if ((buf[i] & 0xE0) === 0xC0) { + this[kMissingBytes] = 2; + } else if ((buf[i] & 0xF0) === 0xE0) { + this[kMissingBytes] = 3; + } else if ((buf[i] & 0xF8) === 0xF0) { + this[kMissingBytes] = 4; + } else { + // invalid + this[kBufferedBytes] = 0; + break; + } -/* - * Returns all complete UTF-8 characters in a Buffer. If the Buffer ended on a - * partial character, the character's bytes are buffered until the required - * number of bytes are available. - */ -function utf8Text(this: StringDecoderBase, buf: Buffer, i: number): string { - const total = utf8CheckIncomplete(this, buf, i); - if (!this.lastNeed) return buf.toString("utf8", i); - this.lastTotal = total; - const end = buf.length - (total - this.lastNeed); - buf.copy(this.lastChar, 0, end); - return buf.toString("utf8", i, end); -} + if (this[kBufferedBytes] >= this[kMissingBytes]) { + // We have enough trailing bytes to complete + // the char + this[kMissingBytes] = 0; + this[kBufferedBytes] = 0; + } -/* - * For UTF-8, a replacement character is added when ending on a partial - * character. - */ -function utf8End(this: Utf8Decoder, buf?: Buffer): string { - const r = buf && buf.length ? this.write(buf) : ""; - if (this.lastNeed) return r + "\ufffd"; - return r; -} + this[kMissingBytes] -= this[kBufferedBytes]; + break; + } + } + } else if (enc === Encoding.Utf16) { + if ((buf.length - bufIdx) % 2 === 1) { + // Have half of a code unit + this[kBufferedBytes] = 1; + this[kMissingBytes] = 1; + } else if ((buf[buf.length - 1] & 0xFC) === 0xD8) { + // 2 bytes out of a 4 byte UTF-16 char + this[kBufferedBytes] = 2; + this[kMissingBytes] = 2; + } + } else if (enc === Encoding.Base64) { + this[kBufferedBytes] = (buf.length - bufIdx) % 3; + if (this[kBufferedBytes] > 0) { + this[kMissingBytes] = 3 - this[kBufferedBytes]; + } + } -function utf8Write( - this: Utf8Decoder | Base64Decoder, - buf: Buffer | string, -): string { - if (typeof buf === "string") { - return buf; - } - if (buf.length === 0) return ""; - let r; - let i; - // Because `TypedArray` is recognized as `ArrayBuffer` but in the reality, there are some fundamental difference. We would need to cast it properly - const normalizedBuffer: Buffer = isBufferType(buf) ? buf : Buffer.from(buf); - if (this.lastNeed) { - r = this.fillLast(normalizedBuffer); - if (r === undefined) return ""; - i = this.lastNeed; - this.lastNeed = 0; - } else { - i = 0; - } - if (i < buf.length) { - return r - ? r + this.text(normalizedBuffer, i) - : this.text(normalizedBuffer, i); - } - return r || ""; -} + if (this[kBufferedBytes] > 0) { + // Copy the bytes that make up the incomplete char + // from the end of the buffer into our `lastChar` buffer + buf.copy( + this.lastChar, + 0, + buf.length - this[kBufferedBytes], + ); + bufEnd -= this[kBufferedBytes]; + } + + rest = bufferToString(buf, this.encoding, bufIdx, bufEnd); + } -function base64Text(this: StringDecoderBase, buf: Buffer, i: number): string { - const n = (buf.length - i) % 3; - if (n === 0) return buf.toString("base64", i); - this.lastNeed = 3 - n; - this.lastTotal = 3; - if (n === 1) { - this.lastChar[0] = buf[buf.length - 1]; + if (prepend.length === 0) { + return rest; + } else { + return prepend + rest; + } } else { - this.lastChar[0] = buf[buf.length - 2]; - this.lastChar[1] = buf[buf.length - 1]; + return bufferToString(buf, this.encoding, bufIdx, bufEnd); } - return buf.toString("base64", i, buf.length - n); } -function base64End(this: Base64Decoder, buf?: Buffer): string { - const r = buf && buf.length ? this.write(buf) : ""; - if (this.lastNeed) { - return r + this.lastChar.toString("base64", 0, 3 - this.lastNeed); +function flush(this: StringDecoder) { + const enc = this.enc; + + if (enc === Encoding.Utf16 && this[kBufferedBytes] % 2 === 1) { + // ignore trailing byte if it isn't a complete code unit (2 bytes) + this[kBufferedBytes] -= 1; + this[kMissingBytes] -= 1; } - return r; -} -function simpleWrite( - this: StringDecoderBase, - buf: Buffer | string, -): string { - if (typeof buf === "string") { - return buf; + if (this[kBufferedBytes] === 0) { + return ""; } - return buf.toString(this.encoding); -} -function simpleEnd(this: GenericDecoder, buf?: Buffer): string { - return buf && buf.length ? this.write(buf) : ""; -} + const ret = bufferToString( + this.lastChar, + this.encoding, + 0, + this[kBufferedBytes], + ); -class StringDecoderBase { - public lastChar: Buffer; - public lastNeed = 0; - public lastTotal = 0; - constructor(public encoding: string, nb: number) { - this.lastChar = Buffer.allocUnsafe(nb); - } -} + this[kBufferedBytes] = 0; + this[kMissingBytes] = 0; -class Base64Decoder extends StringDecoderBase { - public end = base64End; - public fillLast = utf8FillLastIncomplete; - public text = base64Text; - public write = utf8Write; + return ret; +} - constructor(encoding?: string) { - super(normalizeEncoding(encoding), 3); - } +enum Encoding { + Utf8, + Base64, + Utf16, + Ascii, + Latin1, + Hex, } -class GenericDecoder extends StringDecoderBase { - public end = simpleEnd; - public fillLast = undefined; - public text = utf8Text; - public write = simpleWrite; +const kBufferedBytes = Symbol("bufferedBytes"); +const kMissingBytes = Symbol("missingBytes"); - constructor(encoding?: string) { - super(normalizeEncoding(encoding), 4); - } -} +type StringDecoder = { + encoding: string; + end: (buf: Buffer) => string; + write: (buf: Buffer) => string; + lastChar: Buffer; + lastNeed: number; + lastTotal: number; + text: (buf: Buffer, idx: number) => string; + enc: Encoding; -class Utf8Decoder extends StringDecoderBase { - public end = utf8End; - public fillLast = utf8FillLastComplete; - public text = utf8Text; - public write = utf8Write; + decode: (buf: Buffer) => string; - constructor(encoding?: string) { - super(normalizeEncoding(encoding), 4); - } -} + [kBufferedBytes]: number; + [kMissingBytes]: number; + + flush: () => string; +}; /* * StringDecoder provides an interface for efficiently splitting a series of * buffers into a series of JS strings without breaking apart multi-byte * characters. */ -export class StringDecoder { - public encoding: string; - public end: (buf?: Buffer) => string; - public fillLast: ((buf: Buffer) => string | undefined) | undefined; - public lastChar: Buffer; - public lastNeed: number; - public lastTotal: number; - public text: (buf: Buffer, n: number) => string; - public write: (buf: Buffer) => string; - - constructor(encoding?: string) { - const normalizedEncoding = normalizeEncoding(encoding); - let decoder: Utf8Decoder | Base64Decoder | GenericDecoder; - switch (normalizedEncoding) { - case "utf8": - decoder = new Utf8Decoder(encoding); - break; - case "base64": - decoder = new Base64Decoder(encoding); - break; - default: - decoder = new GenericDecoder(encoding); - } - this.encoding = decoder.encoding; - this.end = decoder.end; - this.fillLast = decoder.fillLast; - this.lastChar = decoder.lastChar; - this.lastNeed = decoder.lastNeed; - this.lastTotal = decoder.lastTotal; - this.text = decoder.text; - this.write = decoder.write; +export function StringDecoder(this: Partial<StringDecoder>, encoding?: string) { + const normalizedEncoding = normalizeEncoding(encoding); + let enc: Encoding = Encoding.Utf8; + let bufLen = 0; + switch (normalizedEncoding) { + case "utf8": + enc = Encoding.Utf8; + bufLen = 4; + break; + case "base64": + enc = Encoding.Base64; + bufLen = 3; + break; + case "utf16le": + enc = Encoding.Utf16; + bufLen = 4; + break; + case "hex": + enc = Encoding.Hex; + bufLen = 0; + break; + case "latin1": + enc = Encoding.Latin1; + bufLen = 0; + break; + case "ascii": + enc = Encoding.Ascii; + bufLen = 0; + break; } + this.encoding = normalizedEncoding; + this.lastChar = Buffer.allocUnsafe(bufLen); + this.enc = enc; + this[kBufferedBytes] = 0; + this[kMissingBytes] = 0; + this.flush = flush; + this.decode = decode; } -// Allow calling StringDecoder() without new -const PStringDecoder = new Proxy(StringDecoder, { - apply(_target, thisArg, args) { - // @ts-ignore tedious to replicate types ... - return Object.assign(thisArg, new StringDecoder(...args)); + +/** + * Returns a decoded string, omitting any incomplete multi-bytes + * characters at the end of the Buffer, or TypedArray, or DataView + */ +StringDecoder.prototype.write = function write(buf: Buffer): string { + if (typeof buf === "string") { + return buf; + } + const normalizedBuf = normalizeBuffer(buf); + if (this[kBufferedBytes] === undefined) { + throw new ERR_INVALID_THIS("StringDecoder"); + } + return this.decode(normalizedBuf); +}; + +/** + * Returns any remaining input stored in the internal buffer as a string. + * After end() is called, the stringDecoder object can be reused for new + * input. + */ +StringDecoder.prototype.end = function end(buf: Buffer): string { + let ret = ""; + if (buf !== undefined) { + ret = this.write(buf); + } + if (this[kBufferedBytes] > 0) { + ret += this.flush(); + } + return ret; +}; + +// Below is undocumented but accessible stuff from node's old impl +// (node's tests assert on these, so we need to support them) +StringDecoder.prototype.text = function text( + buf: Buffer, + offset: number, +): string { + this[kBufferedBytes] = 0; + this[kMissingBytes] = 0; + return this.write(buf.subarray(offset)); +}; + +ObjectDefineProperties(StringDecoder.prototype, { + lastNeed: { + configurable: true, + enumerable: true, + get(this: StringDecoder): number { + return this[kMissingBytes]; + }, + }, + lastTotal: { + configurable: true, + enumerable: true, + get(this: StringDecoder): number { + return this[kBufferedBytes] + this[kMissingBytes]; + }, }, }); -export default { StringDecoder: PStringDecoder }; +export default { StringDecoder }; diff --git a/tests/node_compat/config.jsonc b/tests/node_compat/config.jsonc index 4d89b1a89..bf254faf6 100644 --- a/tests/node_compat/config.jsonc +++ b/tests/node_compat/config.jsonc @@ -614,6 +614,7 @@ "test-stream3-cork-uncork.js", "test-stream3-pause-then-read.js", "test-streams-highwatermark.js", + "test-string-decoder.js", "test-timers-api-refs.js", "test-timers-args.js", "test-timers-clear-null-does-not-throw-error.js", diff --git a/tests/node_compat/test.ts b/tests/node_compat/test.ts index bafb14db2..04a85f113 100644 --- a/tests/node_compat/test.ts +++ b/tests/node_compat/test.ts @@ -82,6 +82,7 @@ async function runTest(t: Deno.TestContext, path: string): Promise<void> { "-A", "--quiet", //"--unsafely-ignore-certificate-errors", + "--unstable-unsafe-proto", "--unstable-bare-node-builtins", "--v8-flags=" + v8Flags.join(), "runner.ts", diff --git a/tests/node_compat/test/parallel/test-string-decoder.js b/tests/node_compat/test/parallel/test-string-decoder.js new file mode 100644 index 000000000..84ac71aac --- /dev/null +++ b/tests/node_compat/test/parallel/test-string-decoder.js @@ -0,0 +1,292 @@ +// deno-fmt-ignore-file +// deno-lint-ignore-file + +// Copyright Joyent and Node contributors. All rights reserved. MIT license. +// Taken from Node 18.12.1 +// This file is automatically generated by `tools/node_compat/setup.ts`. Do not modify this file manually. + +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +'use strict'; +const common = require('../common'); +const assert = require('assert'); +const inspect = require('util').inspect; +const StringDecoder = require('string_decoder').StringDecoder; + +// Test default encoding +let decoder = new StringDecoder(); +assert.strictEqual(decoder.encoding, 'utf8'); + +// Should work without 'new' keyword +const decoder2 = {}; +StringDecoder.call(decoder2); +assert.strictEqual(decoder2.encoding, 'utf8'); + +// UTF-8 +test('utf-8', Buffer.from('$', 'utf-8'), '$'); +test('utf-8', Buffer.from('¢', 'utf-8'), '¢'); +test('utf-8', Buffer.from('€', 'utf-8'), '€'); +test('utf-8', Buffer.from('𤭢', 'utf-8'), '𤭢'); +// A mixed ascii and non-ascii string +// Test stolen from deps/v8/test/cctest/test-strings.cc +// U+02E4 -> CB A4 +// U+0064 -> 64 +// U+12E4 -> E1 8B A4 +// U+0030 -> 30 +// U+3045 -> E3 81 85 +test( + 'utf-8', + Buffer.from([0xCB, 0xA4, 0x64, 0xE1, 0x8B, 0xA4, 0x30, 0xE3, 0x81, 0x85]), + '\u02e4\u0064\u12e4\u0030\u3045' +); + +// Some invalid input, known to have caused trouble with chunking +// in https://github.com/nodejs/node/pull/7310#issuecomment-226445923 +// 00: |00000000 ASCII +// 41: |01000001 ASCII +// B8: 10|111000 continuation +// CC: 110|01100 two-byte head +// E2: 1110|0010 three-byte head +// F0: 11110|000 four-byte head +// F1: 11110|001'another four-byte head +// FB: 111110|11 "five-byte head", not UTF-8 +test('utf-8', Buffer.from('C9B5A941', 'hex'), '\u0275\ufffdA'); +test('utf-8', Buffer.from('E2', 'hex'), '\ufffd'); +test('utf-8', Buffer.from('E241', 'hex'), '\ufffdA'); +test('utf-8', Buffer.from('CCCCB8', 'hex'), '\ufffd\u0338'); +test('utf-8', Buffer.from('F0B841', 'hex'), '\ufffdA'); +test('utf-8', Buffer.from('F1CCB8', 'hex'), '\ufffd\u0338'); +test('utf-8', Buffer.from('F0FB00', 'hex'), '\ufffd\ufffd\0'); +test('utf-8', Buffer.from('CCE2B8B8', 'hex'), '\ufffd\u2e38'); +test('utf-8', Buffer.from('E2B8CCB8', 'hex'), '\ufffd\u0338'); +test('utf-8', Buffer.from('E2FBCC01', 'hex'), '\ufffd\ufffd\ufffd\u0001'); +test('utf-8', Buffer.from('CCB8CDB9', 'hex'), '\u0338\u0379'); +// CESU-8 of U+1D40D + +// V8 has changed their invalid UTF-8 handling, see +// https://chromium-review.googlesource.com/c/v8/v8/+/671020 for more info. +test('utf-8', Buffer.from('EDA0B5EDB08D', 'hex'), + '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'); + +// UCS-2 +test('ucs2', Buffer.from('ababc', 'ucs2'), 'ababc'); + +// UTF-16LE +test('utf16le', Buffer.from('3DD84DDC', 'hex'), '\ud83d\udc4d'); // thumbs up + +// Additional UTF-8 tests +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('E1', 'hex')), ''); + +// A quick test for lastChar, lastNeed & lastTotal which are undocumented. +assert(decoder.lastChar.equals(new Uint8Array([0xe1, 0, 0, 0]))); +assert.strictEqual(decoder.lastNeed, 2); +assert.strictEqual(decoder.lastTotal, 3); + +assert.strictEqual(decoder.end(), '\ufffd'); + +// ArrayBufferView tests +const arrayBufferViewStr = 'String for ArrayBufferView tests\n'; +const inputBuffer = Buffer.from(arrayBufferViewStr.repeat(8), 'utf8'); +for (const expectView of common.getArrayBufferViews(inputBuffer)) { + assert.strictEqual( + decoder.write(expectView), + inputBuffer.toString('utf8') + ); + assert.strictEqual(decoder.end(), ''); +} + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('E18B', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ufffd'); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('\ufffd')), '\ufffd'); +assert.strictEqual(decoder.end(), ''); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('\ufffd\ufffd\ufffd')), + '\ufffd\ufffd\ufffd'); +assert.strictEqual(decoder.end(), ''); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('EFBFBDE2', 'hex')), '\ufffd'); +assert.strictEqual(decoder.end(), '\ufffd'); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('F1', 'hex')), ''); +assert.strictEqual(decoder.write(Buffer.from('41F2', 'hex')), '\ufffdA'); +assert.strictEqual(decoder.end(), '\ufffd'); + +// Additional utf8Text test +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.text(Buffer.from([0x41]), 2), ''); + +// Additional UTF-16LE surrogate pair tests +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(Buffer.from('3DD8', 'hex')), ''); +assert.strictEqual(decoder.write(Buffer.from('4D', 'hex')), ''); +assert.strictEqual(decoder.write(Buffer.from('DC', 'hex')), '\ud83d\udc4d'); +assert.strictEqual(decoder.end(), ''); + +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(Buffer.from('3DD8', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ud83d'); + +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(Buffer.from('3DD8', 'hex')), ''); +assert.strictEqual(decoder.write(Buffer.from('4D', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ud83d'); + +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(Buffer.from('3DD84D', 'hex')), '\ud83d'); +assert.strictEqual(decoder.end(), ''); + +// Regression test for https://github.com/nodejs/node/issues/22358 +// (unaligned UTF-16 access). +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(Buffer.alloc(1)), ''); +assert.strictEqual(decoder.write(Buffer.alloc(20)), '\0'.repeat(10)); +assert.strictEqual(decoder.write(Buffer.alloc(48)), '\0'.repeat(24)); +assert.strictEqual(decoder.end(), ''); + +// Regression tests for https://github.com/nodejs/node/issues/22626 +// (not enough replacement chars when having seen more than one byte of an +// incomplete multibyte characters). +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('f69b', 'hex')), ''); +assert.strictEqual(decoder.write(Buffer.from('d1', 'hex')), '\ufffd\ufffd'); +assert.strictEqual(decoder.end(), '\ufffd'); +assert.strictEqual(decoder.write(Buffer.from('f4', 'hex')), ''); +assert.strictEqual(decoder.write(Buffer.from('bde5', 'hex')), '\ufffd\ufffd'); +assert.strictEqual(decoder.end(), '\ufffd'); + +assert.throws( + () => new StringDecoder(1), + { + code: 'ERR_UNKNOWN_ENCODING', + name: 'TypeError', + message: 'Unknown encoding: 1' + } +); + +assert.throws( + () => new StringDecoder('test'), + { + code: 'ERR_UNKNOWN_ENCODING', + name: 'TypeError', + message: 'Unknown encoding: test' + } +); + +assert.throws( + () => new StringDecoder('utf8').write(null), + { + code: 'ERR_INVALID_ARG_TYPE', + name: 'TypeError', + message: 'The "buf" argument must be an instance of Buffer, TypedArray,' + + ' or DataView. Received null' + } +); + +if (common.enoughTestMem) { + assert.throws( + () => new StringDecoder().write(Buffer.alloc((process.arch === 'ia32' ? 0x18ffffe8 : 0x1fffffe8) + 1).fill('a')), + { + code: 'ERR_STRING_TOO_LONG', + } + ); +} + +assert.throws( + () => new StringDecoder('utf8').__proto__.write(Buffer.from('abc')), // eslint-disable-line no-proto + { + code: 'ERR_INVALID_THIS', + } +); + +// Test verifies that StringDecoder will correctly decode the given input +// buffer with the given encoding to the expected output. It will attempt all +// possible ways to write() the input buffer, see writeSequences(). The +// singleSequence allows for easy debugging of a specific sequence which is +// useful in case of test failures. +function test(encoding, input, expected, singleSequence) { + let sequences; + if (!singleSequence) { + sequences = writeSequences(input.length); + } else { + sequences = [singleSequence]; + } + const hexNumberRE = /.{2}/g; + sequences.forEach((sequence) => { + const decoder = new StringDecoder(encoding); + let output = ''; + sequence.forEach((write) => { + output += decoder.write(input.slice(write[0], write[1])); + }); + output += decoder.end(); + if (output !== expected) { + const message = + `Expected "${unicodeEscape(expected)}", ` + + `but got "${unicodeEscape(output)}"\n` + + `input: ${input.toString('hex').match(hexNumberRE)}\n` + + `Write sequence: ${JSON.stringify(sequence)}\n` + + `Full Decoder State: ${inspect(decoder)}`; + assert.fail(message); + } + }); +} + +// unicodeEscape prints the str contents as unicode escape codes. +function unicodeEscape(str) { + let r = ''; + for (let i = 0; i < str.length; i++) { + r += `\\u${str.charCodeAt(i).toString(16)}`; + } + return r; +} + +// writeSequences returns an array of arrays that describes all possible ways a +// buffer of the given length could be split up and passed to sequential write +// calls. +// +// e.G. writeSequences(3) will return: [ +// [ [ 0, 3 ] ], +// [ [ 0, 2 ], [ 2, 3 ] ], +// [ [ 0, 1 ], [ 1, 3 ] ], +// [ [ 0, 1 ], [ 1, 2 ], [ 2, 3 ] ] +// ] +function writeSequences(length, start, sequence) { + if (start === undefined) { + start = 0; + sequence = []; + } else if (start === length) { + return [sequence]; + } + let sequences = []; + for (let end = length; end > start; end--) { + const subSequence = sequence.concat([[start, end]]); + const subSequences = writeSequences(length, end, subSequence, sequences); + sequences = sequences.concat(subSequences); + } + return sequences; +} diff --git a/tools/node_compat/TODO.md b/tools/node_compat/TODO.md index 88cadfc1c..3d8306988 100644 --- a/tools/node_compat/TODO.md +++ b/tools/node_compat/TODO.md @@ -2184,7 +2184,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-stream3-pipeline-async-iterator.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-stream3-pipeline-async-iterator.js) - [parallel/test-string-decoder-end.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-string-decoder-end.js) - [parallel/test-string-decoder-fuzz.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-string-decoder-fuzz.js) -- [parallel/test-string-decoder.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-string-decoder.js) - [parallel/test-stringbytes-external.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-stringbytes-external.js) - [parallel/test-structuredClone-global.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-structuredClone-global.js) - [parallel/test-sync-fileread.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-sync-fileread.js) |