diff --git a/packages/csv-parse/lib/api/index.js b/packages/csv-parse/lib/api/index.js index ca97838cd..f17bc1af8 100644 --- a/packages/csv-parse/lib/api/index.js +++ b/packages/csv-parse/lib/api/index.js @@ -782,8 +782,20 @@ const transform = function (original_options = {}) { } return [undefined, field]; }, + __isCharBoundary: function (pos) { + // UTF-16LE tokens must start on a code unit, including after buffering + // an odd number of bytes or discovering the encoding from a BOM. + const { encoding } = this.options; + return ( + encoding === "utf8" || + encoding === null || + !/^(?:utf-?16le|ucs-?2)$/i.test(encoding) || + (this.state.bufBytesStart + pos) % 2 === 0 + ); + }, __compareBytes: function (sourceBuf, targetBuf, targetPos, firstByte) { - if (sourceBuf[0] !== firstByte) return 0; + if (sourceBuf[0] !== firstByte || !this.__isCharBoundary(targetPos)) + return 0; const sourceLength = sourceBuf.length; for (let i = 1; i < sourceLength; i++) { if (sourceBuf[i] !== targetBuf[targetPos + i]) return 0; @@ -841,7 +853,7 @@ const transform = function (original_options = {}) { return false; } } - return true; + return this.__isCharBoundary(pos); } return false; }, @@ -863,7 +875,7 @@ const transform = function (original_options = {}) { return false; } } - return true; + return this.__isCharBoundary(pos); }, __isRecordDelimiter: function (chr, buf, pos) { const { record_delimiter } = this.options; diff --git a/packages/csv-parse/test/option.comment.utf16le.js b/packages/csv-parse/test/option.comment.utf16le.js new file mode 100644 index 000000000..f3c7f3861 --- /dev/null +++ b/packages/csv-parse/test/option.comment.utf16le.js @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { parse } from "../lib/index.js"; +import { parse as parseSync } from "../lib/sync.js"; + +describe("Option `comment` with UTF-16LE", function () { + for (const encoding of ["utf16le", "utf-16le", "ucs2", "ucs-2", "UTF16LE"]) { + it(`ignores unaligned comment bytes with ${encoding}`, function () { + // The bytes 23 00 span two characters, rather than encoding a '#'. + const value = "\u2301\u0100"; + const input = Buffer.from(`${value},b# ignored|# comment|c,d`, encoding); + assert.deepEqual( + parseSync(input, { encoding, comment: "#", record_delimiter: "|" }), + [ + [value, "b"], + ["c", "d"], + ], + ); + }); + } + + it("ignores unaligned multi-character comments", function () { + const value = "\u2301\u2100\u0100"; + assert.deepEqual( + parseSync(Buffer.from(`${value},b#! ignored|c,d`, "utf16le"), { + encoding: "utf16le", + comment: "#!", + record_delimiter: "|", + }), + [ + [value, "b"], + ["c", "d"], + ], + ); + }); + + it("uses the BOM-detected encoding with byte-sized writes", function (next) { + const value = "\u2301\u0100"; + const input = Buffer.from(`\ufeff${value},b# ignored|c,d`, "utf16le"); + const parser = parse( + { bom: true, comment: "#", record_delimiter: "|" }, + (err, records) => { + if (err) return next(err); + assert.deepEqual(records, [ + [value, "b"], + ["c", "d"], + ]); + next(); + }, + ); + for (let i = 0; i < input.length; i++) { + parser.write(input.subarray(i, i + 1)); + } + parser.end(); + }); + + it("uses the absolute byte position across chunk boundaries", async function () { + const value = "prefix\u2301\u0100suffix"; + const input = Buffer.from(`${value},b# ignored|c,d`, "utf16le"); + for (let split = 1; split < input.length; split++) { + await new Promise((resolve, reject) => { + const parser = parse( + { encoding: "utf16le", comment: "#", record_delimiter: "|" }, + (err, records) => { + if (err) return reject(err); + try { + assert.deepEqual(records, [ + [value, "b"], + ["c", "d"], + ]); + resolve(); + } catch (error) { + reject(error); + } + }, + ); + parser.write(input.subarray(0, split)); + parser.end(input.subarray(split)); + }); + } + }); + + it("continues to match raw binary comment buffers at any byte position", function () { + assert.deepEqual( + parseSync(Buffer.from("a#\0 ignored|c"), { + encoding: null, + comment: Buffer.from("#\0"), + record_delimiter: "|", + }), + [[Buffer.from("a")], [Buffer.from("c")]], + ); + }); +}); diff --git a/packages/csv-parse/test/option.quote.utf16le.js b/packages/csv-parse/test/option.quote.utf16le.js new file mode 100644 index 000000000..30e58a3c3 --- /dev/null +++ b/packages/csv-parse/test/option.quote.utf16le.js @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { parse } from "../lib/index.js"; +import { parse as parseSync } from "../lib/sync.js"; + +describe("UTF-16LE quote and escape boundaries", function () { + const quoteValue = "\u2201\u0100"; + const escapeValue = "\u5c01\u0100"; + for (const encoding of ["utf16le", "utf-16le", "ucs2", "ucs-2", "UTF16LE"]) { + for (const quoted of [false, true]) { + it(`preserves unaligned quote bytes with ${encoding} and quoted=${quoted}`, function () { + const field = quoted ? `"${quoteValue}"` : quoteValue; + assert.deepEqual( + parseSync(Buffer.from(`${field},b`, encoding), { encoding }), + [[quoteValue, "b"]], + ); + }); + } + } + + it("preserves unaligned custom escape bytes", function () { + assert.deepEqual( + parseSync(Buffer.from(`"${escapeValue}",b`, "utf16le"), { + encoding: "utf16le", + escape: "\\", + }), + [[escapeValue, "b"]], + ); + }); + + it("uses absolute positions across every stream split", async function () { + const input = Buffer.from( + `"prefix${quoteValue}${escapeValue}",b`, + "utf16le", + ); + for (let split = 1; split < input.length; split++) { + const records = await new Promise((resolve, reject) => { + const parser = parse( + { encoding: "utf16le", escape: "\\" }, + (err, records) => { + if (err) return reject(err); + resolve(records); + }, + ); + parser.write(input.subarray(0, split)); + parser.end(input.subarray(split)); + }); + assert.deepEqual(records, [[`prefix${quoteValue}${escapeValue}`, "b"]]); + } + }); + + it("continues to parse real quotes and escaped quotes", function () { + assert.deepEqual( + parseSync(Buffer.from('"a""b",c', "utf16le"), { encoding: "utf16le" }), + [['a"b', "c"]], + ); + assert.deepEqual( + parseSync(Buffer.from('"a\\"b",c', "utf16le"), { + encoding: "utf16le", + escape: "\\", + }), + [['a"b', "c"]], + ); + }); +});