feat(lexer): prototype versioned detection baselines

Makefile

@@ -6,7 +6,7 @@ WASM_OPT ?= ../binaryen/bin/wasm-opt
 lib/lexer.wat: lib/lexer.wasm
 	$(WASM2WAT) lib/lexer.wasm -o lib/lexer.wat
 
-lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c | lib/
+lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c src/reexport-analysis.h | lib/
 	node build/wasm.js --docker
 
 lib/:

README.md

@@ -16,7 +16,7 @@ This project is used in Node.js core for detecting the named exports available w
 
 PRs will be accepted and upstreamed for parser bugs, performance improvements or new syntax support only.
 
-_Detection patterns for this project are **frozen**_. This is because adding any new export detection patterns would result in fragmented backwards-compatibility. Specifically, it would be very difficult to figure out why an ES module named export for CommonJS might work in newer Node.js versions but not older versions. This problem would only be discovered downstream of module authors, with the fix for module authors being to then have to understand which patterns in this project provide full backwards-compatibily. Rather, by fully freezing the detected patterns, if it works in any Node.js version it will work in any other. Build tools can also reliably treat the supported syntax for this project as a part of their output target for ensuring syntax support.
+_Detection patterns for the `legacy` baseline are **frozen**_. This is because adding any new export detection patterns would result in fragmented backwards-compatibility. Specifically, it would be very difficult to figure out why an ES module named export for CommonJS might work in newer Node.js versions but not older versions. This problem would only be discovered downstream of module authors, with the fix for module authors being to then have to understand which patterns in this project provide full backwards-compatibily. Rather, by fully freezing the detected patterns, if it works in any Node.js version it will work in any other. Build tools can also reliably treat the supported syntax for this project as a part of their output target for ensuring syntax support.
 
 ### Usage
 
@@ -29,30 +29,16 @@ For use in CommonJS:
 ```js
 const { parse } = require('cjs-module-lexer');
 
-// `init` return a promise for parity with the ESM API, but you do not have to call it
+// Parsing initializes Wasm on demand. Await init() to initialize it in advance.
 
-const { exports, reexports } = parse(`
-  // named exports detection
-  module.exports.a = 'a';
-  (function () {
-    exports.b = 'b';
-  })();
-  Object.defineProperty(exports, 'c', { value: 'c' });
-  /* exports.d = 'not detected'; */
-
-  // reexports detection
-  if (maybe) module.exports = require('./dep1.js');
-  if (another) module.exports = require('./dep2.js');
-
-  // literal exports assignments
-  module.exports = { a, b: c, d, 'e': f }
-
-  // __esModule detection
-  Object.defineProperty(module.exports, '__esModule', { value: true })
+const { exports, reexports, complete } = parse(`
+  if (mode) module.exports = { own: 1, ...require('./dep1.js') };
+  else module.exports = { own: 2, ...require('./dep2.js') };
 `);
 
-// exports === ['a', 'b', 'c', '__esModule']
-// reexports === ['./dep1.js', './dep2.js']
+// exports: ['own']
+// reexports: [['./dep1.js'], ['./dep2.js']]
+// complete: true
 ```
 
 When using the ESM version, Wasm is supported instead:
@@ -69,10 +55,52 @@ Call `initSync()` directly to choose when the blocking initialization runs.
 Browser main threads can restrict synchronous Wasm compilation, so await
 `init()` before parsing in browsers.
 
-The Wasm build is around 1.5x faster and without a cold start.
+### Detection baselines (proof of concept)
+
+The default `reexports` value stays flat for one distinct final dependency list.
+Different conditional lists produce an array of arrays.
+Each inner array contains dependencies combined in one final export value.
+The outer array contains alternative final values.
+Identical lists are deduplicated before choosing the return shape.
+Conditional nesting changes the return type and requires a major release.
+
+```js
+const { reexports, complete } = parse(`
+  if (production) module.exports = {...require('./a.cjs'), ...require('./b.cjs')};
+  else module.exports = {...require('./c.cjs'), ...require('./d.cjs')};
+`);
+// reexports: [['./a.cjs', './b.cjs'], ['./c.cjs', './d.cjs']]
+// complete: true
+```
+
+Sequential replacements retain only the final value.
+Pure guards remove unreachable branches. Unknown guards retain their possible alternatives.
+Object spreads of direct `require` calls combine dependencies in one group.
+The `exports` array contains detected literal names from surviving values.
+
+Flow analysis runs only after the lexer detects more than one `module.exports` replacement.
+The counter ignores comments, strings, property writes, and comparisons.
+With fewer replacements, existing detection patterns run and their reexports stay flat.
+The optional third argument selects a versioned detection baseline.
+Select `legacy` for frozen detection, or `flow-v1` for the default conditional analysis.
+A baseline can combine detection rules without an option for each rule.
+
+The C lexer compiled to Wasm owns flow analysis.
+The JavaScript entry point invokes it only for competing replacements.
+Use valid CommonJS source with standard CommonJS bindings and an ordinary writable `module.exports` property.
+Analysis describes successful synchronous completion and does not execute source or resolve dependencies.
+Opaque reads and dependency execution invalidate earlier exposed values. A fresh replacement can establish a known value again.
+
+Results include `complete` only when flow analysis runs.
+Unsupported syntax, effects, bindings, filtered string literals, and analysis limits make it false.
+An incomplete result contains confirmed alternatives and omits unknown values.
+Token, nesting, outcome, and work limits bound the analyzer.
+Existing lexer errors still apply.
 
 ### Grammar
 
+The grammar below describes the `legacy` baseline.
+
 CommonJS exports matches are run against the source token stream.
 
 The token grammar is:

build/Makefile

@@ -7,11 +7,11 @@ WASM_LDFLAGS := -nostartfiles
 WASM_TARGET := -target wasm32-unknown-wasi
 WASM_EXTRA_CFLAGS := -I include-wasm/ -Wno-logical-op-parentheses -Wno-parentheses -Oz
 WASM_EXTRA_LDFLAGS := -Wl,-z,stack-size=13312,--no-entry,--compress-relocations,--strip-all
-WASM_EXTRA_LDFLAGS += -Wl,--export=__heap_base,--export=parseCJS,--export=sa
+WASM_EXTRA_LDFLAGS += -Wl,--export=__heap_base,--export=parseCJS,--export=parseCJSGrouped,--export=sa
 WASM_EXTRA_LDFLAGS += -Wl,--export=e,--export=re,--export=es,--export=ee
 WASM_EXTRA_LDFLAGS += -Wl,--export=rre,--export=ree,--export=res,--export=ru,--export=us,--export=ue
 
-lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c
+lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c src/reexport-analysis.h
 	@mkdir -p lib
 	$(WASM_CC) $(WASM_CFLAGS) $(WASM_TARGET) $(WASM_EXTRA_CFLAGS) \
         src/lexer.c -o lib/lexer.wasm \

include-wasm/cjs-module-lexer.h

@@ -72,6 +72,8 @@ uint32_t ee () {
 }
 // getReexportStart
 uint32_t res () {
+  // A null slice starts a group; Wasm exposes this sentinel as signed -1.
+  if (reexport_read_head->start == NULL) return UINT32_MAX;
   return reexport_read_head->start - source;
 }
 // getReexportEnd

lexer.d.ts

@@ -1,8 +1,23 @@
 export interface Exports {
+  exports: string[];
+  /** Flat dependencies, or alternative lists when conditional final dependencies differ. */
+  reexports: string[] | string[][];
+  /** Present when competing replacements are analyzed. False for unsupported effects, filtered names, or limits. */
+  complete?: boolean;
+}
+
+export interface LegacyExports {
   exports: string[];
   reexports: string[];
 }
 
+export interface ParseOptions {
+  baseline?: 'legacy' | 'flow-v1';
+}
+
+export declare function parse(source: string, name: string | undefined, options: { baseline: 'legacy' }): LegacyExports;
+export declare function parse(source: string, name: string | undefined, options?: { baseline?: 'flow-v1' }): Exports;
+export declare function parse(source: string, name: string | undefined, options?: ParseOptions): Exports | LegacyExports;
 export declare function parse(source: string, name?: string): Exports;
 export declare function init(): Promise<void>;
 export declare function initSync(): void;

lexer.js

@@ -13,24 +13,26 @@ let openTokenDepth,
   lastStarExportSpecifier,
   _exports,
   unsafeGetters,
-  reexports;
+  reexports,
+  moduleExportCount;
 
 function resetState () {
   openTokenDepth = 0;
   templateDepth = -1;
   lastTokenPos = -1;
   lastSlashWasDivision = false;
-  templateStack = new Array(1024);
+  if (templateStack === undefined) templateStack = new Array(1024);
   templateStackDepth = 0;
-  openTokenPosStack = new Array(1024);
-  openClassPosStack = new Array(1024);
+  if (openTokenPosStack === undefined) openTokenPosStack = new Array(1024);
+  if (openClassPosStack === undefined) openClassPosStack = new Array(1024);
   nextBraceIsClass = false;
   starExportMap = Object.create(null);
   lastStarExportSpecifier = null;
 
   _exports = new Set();
   unsafeGetters = new Set();
   reexports = new Set();
+  moduleExportCount = 0;
 }
 
 // RequireType
@@ -52,7 +54,13 @@ function pushTemplate () {
   templateDepth = ++openTokenDepth;
 }
 
-function parseCJS (source, name = '@') {
+/** @param {string} source @param {string} [name] @param {import('./lexer').ParseOptions} [options] */
+function parseCJS (source, name = '@', options) {
+  const baseline = options === undefined ? undefined : options.baseline;
+  if (baseline !== undefined && typeof baseline !== 'string')
+    throw new TypeError('Detection baseline must be a string');
+  if (baseline !== undefined && baseline !== 'legacy' && baseline !== 'flow-v1')
+    throw new RangeError('Unknown detection baseline: ' + baseline);
   resetState();
   try {
     parseSource(source);
@@ -62,6 +70,10 @@ function parseCJS (source, name = '@') {
     e.loc = pos;
     throw e;
   }
+  if (baseline !== 'legacy' && moduleExportCount > 1) {
+    resetState();
+    return require('./dist/lexer.js').parse(source, name, baseline === undefined ? undefined : { baseline });
+  }
   const result = { exports: [..._exports].filter(expt => expt !== undefined && !unsafeGetters.has(expt)), reexports: [...reexports].filter(reexpt => reexpt !== undefined) };
   resetState();
   return result;
@@ -897,6 +909,7 @@ function tryParseModuleExportsDotAssign () {
   pos = revertPos;
 }
 
+/** @param {boolean} assign */
 function tryParseExportsDotAssign (assign) {
   pos += 7;
   const revertPos = pos - 1;
@@ -937,6 +950,8 @@ function tryParseExportsDotAssign (assign) {
     // module.exports =
     case 61/*=*/: {
       if (assign) {
+        if (moduleExportCount < 2 && source.charCodeAt(pos + 1) !== 61)
+          moduleExportCount++;
         if (reexports.size)
           reexports = new Set();
         pos++;
@@ -951,6 +966,13 @@ function tryParseExportsDotAssign (assign) {
         if (ch === 114/*r*/)
           tryParseRequire(ExportAssign);
       }
+      break;
+    }
+    case 38/*&*/:
+    case 124/*|*/:
+    case 63/*?*/: {
+      if (assign && moduleExportCount < 2 && source.charCodeAt(pos + 1) === ch &&
+          source.charCodeAt(pos + 2) === 61) moduleExportCount++;
     }
   }
   pos = revertPos;
@@ -1639,8 +1661,6 @@ function isExpressionTerminator (curPos) {
   return false;
 }
 
-const initPromise = Promise.resolve();
-
-module.exports.init = () => initPromise;
-module.exports.initSync = () => {};
+module.exports.init = () => require('./dist/lexer.js').init();
+module.exports.initSync = () => require('./dist/lexer.js').initSync();
 module.exports.parse = parseCJS;

src/lexer.c

@@ -10,6 +10,7 @@ const uint16_t* EMPTY_CHAR = &__empty_char;
 #define MAX_STAR_EXPORTS 256
 const uint16_t* source;
 
+
 bool lastSlashWasDivision;
 uint16_t templateStackDepth;
 uint16_t openTokenDepth;
@@ -32,6 +33,11 @@ uint16_t* lastReexportEnd;
 uint32_t parse_error;
 uint32_t error = 0;
 uint32_t sourceLen;
+uint32_t moduleExportCount;
+
+#ifdef __wasm__
+#include "reexport-analysis.h"
+#endif
 
 uint16_t templateStack_[STACK_DEPTH];
 uint16_t* openTokenPosStack_[STACK_DEPTH];
@@ -72,6 +78,7 @@ static inline __attribute__((always_inline)) bool pushTemplate () {
 uint32_t parseCJS (uint16_t* _source, uint32_t _sourceLen, void (*_addExport)(const uint16_t*, const uint16_t*), void (*_addReexport)(const uint16_t*, const uint16_t*), void (*_addUnsafeGetter)(const uint16_t*, const uint16_t*), void (*_clearReexports)()) {
   source = _source;
   sourceLen = _sourceLen;
+  moduleExportCount = 0;
   if (_addExport)
     addExport = _addExport;
   if (_addReexport)
@@ -263,7 +270,12 @@ uint32_t parseCJS (uint16_t* _source, uint32_t _sourceLen, void (*_addExport)(co
     return error;
 
   // success
+#ifdef __wasm__
+  // The low byte is reserved for parse errors in the host wrapper.
+  return moduleExportCount << 8;
+#else
   return 0;
+#endif
 }
 
 void tryBacktrackAddStarExportBinding (uint16_t* bPos) {
@@ -939,6 +951,8 @@ void tryParseExportsDotAssign (bool assign) {
     // module.exports =
     case '=': {
       if (assign) {
+        if (moduleExportCount < 2 && pos[1] != '=')
+          moduleExportCount++;
         clearReexports();
         pos++;
         ch = commentWhitespace();
@@ -952,6 +966,13 @@ void tryParseExportsDotAssign (bool assign) {
         if (ch == 'r')
           tryParseRequire(ExportAssign);
       }
+      break;
+    }
+    case '&':
+    case '|':
+    case '?': {
+      if (assign && moduleExportCount < 2 && end - pos >= 2 && pos[1] == ch && pos[2] == '=')
+        moduleExportCount++;
     }
   }
   pos = revertPos;

src/lexer.js

@@ -3,7 +3,13 @@ let wasm;
 const isLE = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
 const hasBuffer = typeof Buffer !== 'undefined';
 
-export function parse (source, name = '@') {
+/** @param {string} source @param {string} [name] @param {import('../lexer').ParseOptions} [options] */
+export function parse (source, name = '@', options) {
+  const baseline = options === undefined ? undefined : options.baseline;
+  if (baseline !== undefined && typeof baseline !== 'string')
+    throw new TypeError('Detection baseline must be a string');
+  if (baseline !== undefined && baseline !== 'legacy' && baseline !== 'flow-v1')
+    throw new RangeError('Unknown detection baseline: ' + baseline);
   if (!wasm)
     initSync();
 
@@ -21,7 +27,9 @@ export function parse (source, name = '@') {
   else
     (isLE ? copyLE : copyBE)(source, new Uint16Array(wasm.memory.buffer, addr, len));
 
-  const err_code = wasm.parseCJS(addr, source.length, 0, 0, 0);
+  const status = baseline === 'legacy' ? wasm.parseCJS(addr, source.length, 0, 0, 0) :
+    wasm.parseCJSGrouped(addr, source.length);
+  const err_code = status & 255;
 
   if (err_code) {
     const err = new Error(`Parse error ${name}${wasm.e()}:${source.slice(0, wasm.e()).split('\n').length}:${wasm.e() - source.lastIndexOf('\n', wasm.e() - 1)}`);
@@ -32,21 +40,49 @@ export function parse (source, name = '@') {
   }
 
   let exports = new Set(), reexports = new Set(), unsafeGetters = new Set();
+  const groups = status & 1024 ? [] : undefined;
+  let complete = !(status & 2048);
 
   while (wasm.rre()) {
-    const reexptStr = decode(source.slice(wasm.res(), wasm.ree()));
+    const start = wasm.res();
+    if (start === -1) {
+      reexports = new Set();
+      groups.push(reexports);
+      continue;
+    }
+    const reexptStr = decode(source.slice(start, wasm.ree()));
     if (reexptStr)
       reexports.add(reexptStr);
+    else if (reexptStr === undefined)
+      complete = false;
   }
   while (wasm.ru())
     unsafeGetters.add(decode(source.slice(wasm.us(), wasm.ue())));
   while (wasm.re()) {
-    let exptStr = decode(source.slice(wasm.es(), wasm.ee()));
+    const text = source.slice(wasm.es(), wasm.ee());
+    let exptStr = status & 1024 && text.charCodeAt(0) >= 48 && text.charCodeAt(0) <= 57 ?
+      String(Number(text)) : decode(text);
     if (exptStr !== undefined && !unsafeGetters.has(exptStr))
       exports.add(exptStr);
+    else if (exptStr === undefined)
+      complete = false;
   }
 
-  return { exports: [...exports], reexports: [...reexports] };
+  if (!(status & 1024)) return { exports: [...exports], reexports: [...reexports] };
+  if (groups.length === 1)
+    return { exports: [...exports], reexports: [...reexports], complete };
+  const alternatives = [];
+  const seen = new Set();
+  let hasDependencies = false;
+  for (const group of groups) {
+    const values = [...group];
+    const key = JSON.stringify(values);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    alternatives.push(values);
+    if (values.length) hasDependencies = true;
+  }
+  return { exports: [...exports], reexports: hasDependencies ? alternatives.length === 1 ? alternatives[0] : alternatives : [], complete };
 }
 
 function decode (str) {