fix(csv-parse): align UTF-16LE comments, quotes and escapes

packages/csv-parse/lib/api/index.js

@@ -782,8 +782,20 @@ const transform = function (original_options = {}) {
       }
       return [undefined, field];
     },
+    __isCharBoundary: function (pos) {
+      // UTF-16LE tokens must start on a code unit, including after buffering
+      // an odd number of bytes or discovering the encoding from a BOM.
+      const { encoding } = this.options;
+      return (
+        encoding === "utf8" ||
+        encoding === null ||
+        !/^(?:utf-?16le|ucs-?2)$/i.test(encoding) ||
+        (this.state.bufBytesStart + pos) % 2 === 0
+      );
+    },
     __compareBytes: function (sourceBuf, targetBuf, targetPos, firstByte) {
-      if (sourceBuf[0] !== firstByte) return 0;
+      if (sourceBuf[0] !== firstByte || !this.__isCharBoundary(targetPos))
+        return 0;
       const sourceLength = sourceBuf.length;
       for (let i = 1; i < sourceLength; i++) {
         if (sourceBuf[i] !== targetBuf[targetPos + i]) return 0;
@@ -841,7 +853,7 @@ const transform = function (original_options = {}) {
             return false;
           }
         }
-        return true;
+        return this.__isCharBoundary(pos);
       }
       return false;
     },
@@ -863,7 +875,7 @@ const transform = function (original_options = {}) {
           return false;
         }
       }
-      return true;
+      return this.__isCharBoundary(pos);
     },
     __isRecordDelimiter: function (chr, buf, pos) {
       const { record_delimiter } = this.options;

packages/csv-parse/test/option.comment.utf16le.js

@@ -0,0 +1,92 @@
+import assert from "node:assert/strict";
+import { parse } from "../lib/index.js";
+import { parse as parseSync } from "../lib/sync.js";
+
+describe("Option `comment` with UTF-16LE", function () {
+  for (const encoding of ["utf16le", "utf-16le", "ucs2", "ucs-2", "UTF16LE"]) {
+    it(`ignores unaligned comment bytes with ${encoding}`, function () {
+      // The bytes 23 00 span two characters, rather than encoding a '#'.
+      const value = "\u2301\u0100";
+      const input = Buffer.from(`${value},b# ignored|# comment|c,d`, encoding);
+      assert.deepEqual(
+        parseSync(input, { encoding, comment: "#", record_delimiter: "|" }),
+        [
+          [value, "b"],
+          ["c", "d"],
+        ],
+      );
+    });
+  }
+
+  it("ignores unaligned multi-character comments", function () {
+    const value = "\u2301\u2100\u0100";
+    assert.deepEqual(
+      parseSync(Buffer.from(`${value},b#! ignored|c,d`, "utf16le"), {
+        encoding: "utf16le",
+        comment: "#!",
+        record_delimiter: "|",
+      }),
+      [
+        [value, "b"],
+        ["c", "d"],
+      ],
+    );
+  });
+
+  it("uses the BOM-detected encoding with byte-sized writes", function (next) {
+    const value = "\u2301\u0100";
+    const input = Buffer.from(`\ufeff${value},b# ignored|c,d`, "utf16le");
+    const parser = parse(
+      { bom: true, comment: "#", record_delimiter: "|" },
+      (err, records) => {
+        if (err) return next(err);
+        assert.deepEqual(records, [
+          [value, "b"],
+          ["c", "d"],
+        ]);
+        next();
+      },
+    );
+    for (let i = 0; i < input.length; i++) {
+      parser.write(input.subarray(i, i + 1));
+    }
+    parser.end();
+  });
+
+  it("uses the absolute byte position across chunk boundaries", async function () {
+    const value = "prefix\u2301\u0100suffix";
+    const input = Buffer.from(`${value},b# ignored|c,d`, "utf16le");
+    for (let split = 1; split < input.length; split++) {
+      await new Promise((resolve, reject) => {
+        const parser = parse(
+          { encoding: "utf16le", comment: "#", record_delimiter: "|" },
+          (err, records) => {
+            if (err) return reject(err);
+            try {
+              assert.deepEqual(records, [
+                [value, "b"],
+                ["c", "d"],
+              ]);
+              resolve();
+            } catch (error) {
+              reject(error);
+            }
+          },
+        );
+        parser.write(input.subarray(0, split));
+        parser.end(input.subarray(split));
+      });
+    }
+  });
+
+  it("continues to match raw binary comment buffers at any byte position", function () {
+    assert.deepEqual(
+      parseSync(Buffer.from("a#\0 ignored|c"), {
+        encoding: null,
+        comment: Buffer.from("#\0"),
+        record_delimiter: "|",
+      }),
+      [[Buffer.from("a")], [Buffer.from("c")]],
+    );
+  });
+});

packages/csv-parse/test/option.quote.utf16le.js

@@ -0,0 +1,64 @@
+import assert from "node:assert/strict";
+import { parse } from "../lib/index.js";
+import { parse as parseSync } from "../lib/sync.js";
+
+describe("UTF-16LE quote and escape boundaries", function () {
+  const quoteValue = "\u2201\u0100";
+  const escapeValue = "\u5c01\u0100";
+  for (const encoding of ["utf16le", "utf-16le", "ucs2", "ucs-2", "UTF16LE"]) {
+    for (const quoted of [false, true]) {
+      it(`preserves unaligned quote bytes with ${encoding} and quoted=${quoted}`, function () {
+        const field = quoted ? `"${quoteValue}"` : quoteValue;
+        assert.deepEqual(
+          parseSync(Buffer.from(`${field},b`, encoding), { encoding }),
+          [[quoteValue, "b"]],
+        );
+      });
+    }
+  }
+
+  it("preserves unaligned custom escape bytes", function () {
+    assert.deepEqual(
+      parseSync(Buffer.from(`"${escapeValue}",b`, "utf16le"), {
+        encoding: "utf16le",
+        escape: "\\",
+      }),
+      [[escapeValue, "b"]],
+    );
+  });
+
+  it("uses absolute positions across every stream split", async function () {
+    const input = Buffer.from(
+      `"prefix${quoteValue}${escapeValue}",b`,
+      "utf16le",
+    );
+    for (let split = 1; split < input.length; split++) {
+      const records = await new Promise((resolve, reject) => {
+        const parser = parse(
+          { encoding: "utf16le", escape: "\\" },
+          (err, records) => {
+            if (err) return reject(err);
+            resolve(records);
+          },
+        );
+        parser.write(input.subarray(0, split));
+        parser.end(input.subarray(split));
+      });
+      assert.deepEqual(records, [[`prefix${quoteValue}${escapeValue}`, "b"]]);
+    }
+  });
+
+  it("continues to parse real quotes and escaped quotes", function () {
+    assert.deepEqual(
+      parseSync(Buffer.from('"a""b",c', "utf16le"), { encoding: "utf16le" }),
+      [['a"b', "c"]],
+    );
+    assert.deepEqual(
+      parseSync(Buffer.from('"a\\"b",c', "utf16le"), {
+        encoding: "utf16le",
+        escape: "\\",
+      }),
+      [['a"b', "c"]],
+    );
+  });
+});