diff --git a/Makefile b/Makefile index bf45b62..726d19b 100755 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ WASM_OPT ?= ../binaryen/bin/wasm-opt lib/lexer.wat: lib/lexer.wasm $(WASM2WAT) lib/lexer.wasm -o lib/lexer.wat -lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c | lib/ +lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c src/reexport-analysis.h | lib/ node build/wasm.js --docker lib/: diff --git a/README.md b/README.md index 18bfa08..d61803b 100755 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ This project is used in Node.js core for detecting the named exports available w PRs will be accepted and upstreamed for parser bugs, performance improvements or new syntax support only. -_Detection patterns for this project are **frozen**_. This is because adding any new export detection patterns would result in fragmented backwards-compatibility. Specifically, it would be very difficult to figure out why an ES module named export for CommonJS might work in newer Node.js versions but not older versions. This problem would only be discovered downstream of module authors, with the fix for module authors being to then have to understand which patterns in this project provide full backwards-compatibily. Rather, by fully freezing the detected patterns, if it works in any Node.js version it will work in any other. Build tools can also reliably treat the supported syntax for this project as a part of their output target for ensuring syntax support. +_Detection patterns for the `legacy` baseline are **frozen**_. This is because adding any new export detection patterns would result in fragmented backwards-compatibility. Specifically, it would be very difficult to figure out why an ES module named export for CommonJS might work in newer Node.js versions but not older versions. This problem would only be discovered downstream of module authors, with the fix for module authors being to then have to understand which patterns in this project provide full backwards-compatibily. Rather, by fully freezing the detected patterns, if it works in any Node.js version it will work in any other. Build tools can also reliably treat the supported syntax for this project as a part of their output target for ensuring syntax support. ### Usage @@ -29,30 +29,16 @@ For use in CommonJS: ```js const { parse } = require('cjs-module-lexer'); -// `init` return a promise for parity with the ESM API, but you do not have to call it +// Parsing initializes Wasm on demand. Await init() to initialize it in advance. -const { exports, reexports } = parse(` - // named exports detection - module.exports.a = 'a'; - (function () { - exports.b = 'b'; - })(); - Object.defineProperty(exports, 'c', { value: 'c' }); - /* exports.d = 'not detected'; */ - - // reexports detection - if (maybe) module.exports = require('./dep1.js'); - if (another) module.exports = require('./dep2.js'); - - // literal exports assignments - module.exports = { a, b: c, d, 'e': f } - - // __esModule detection - Object.defineProperty(module.exports, '__esModule', { value: true }) +const { exports, reexports, complete } = parse(` + if (mode) module.exports = { own: 1, ...require('./dep1.js') }; + else module.exports = { own: 2, ...require('./dep2.js') }; `); -// exports === ['a', 'b', 'c', '__esModule'] -// reexports === ['./dep1.js', './dep2.js'] +// exports: ['own'] +// reexports: [['./dep1.js'], ['./dep2.js']] +// complete: true ``` When using the ESM version, Wasm is supported instead: @@ -69,10 +55,52 @@ Call `initSync()` directly to choose when the blocking initialization runs. Browser main threads can restrict synchronous Wasm compilation, so await `init()` before parsing in browsers. -The Wasm build is around 1.5x faster and without a cold start. +### Detection baselines (proof of concept) + +The default `reexports` value stays flat for one distinct final dependency list. +Different conditional lists produce an array of arrays. +Each inner array contains dependencies combined in one final export value. +The outer array contains alternative final values. +Identical lists are deduplicated before choosing the return shape. +Conditional nesting changes the return type and requires a major release. + +```js +const { reexports, complete } = parse(` + if (production) module.exports = {...require('./a.cjs'), ...require('./b.cjs')}; + else module.exports = {...require('./c.cjs'), ...require('./d.cjs')}; +`); +// reexports: [['./a.cjs', './b.cjs'], ['./c.cjs', './d.cjs']] +// complete: true +``` + +Sequential replacements retain only the final value. +Pure guards remove unreachable branches. Unknown guards retain their possible alternatives. +Object spreads of direct `require` calls combine dependencies in one group. +The `exports` array contains detected literal names from surviving values. + +Flow analysis runs only after the lexer detects more than one `module.exports` replacement. +The counter ignores comments, strings, property writes, and comparisons. +With fewer replacements, existing detection patterns run and their reexports stay flat. +The optional third argument selects a versioned detection baseline. +Select `legacy` for frozen detection, or `flow-v1` for the default conditional analysis. +A baseline can combine detection rules without an option for each rule. + +The C lexer compiled to Wasm owns flow analysis. +The JavaScript entry point invokes it only for competing replacements. +Use valid CommonJS source with standard CommonJS bindings and an ordinary writable `module.exports` property. +Analysis describes successful synchronous completion and does not execute source or resolve dependencies. +Opaque reads and dependency execution invalidate earlier exposed values. A fresh replacement can establish a known value again. + +Results include `complete` only when flow analysis runs. +Unsupported syntax, effects, bindings, filtered string literals, and analysis limits make it false. +An incomplete result contains confirmed alternatives and omits unknown values. +Token, nesting, outcome, and work limits bound the analyzer. +Existing lexer errors still apply. ### Grammar +The grammar below describes the `legacy` baseline. + CommonJS exports matches are run against the source token stream. The token grammar is: diff --git a/build/Makefile b/build/Makefile index b8e96d8..7d78168 100755 --- a/build/Makefile +++ b/build/Makefile @@ -7,11 +7,11 @@ WASM_LDFLAGS := -nostartfiles WASM_TARGET := -target wasm32-unknown-wasi WASM_EXTRA_CFLAGS := -I include-wasm/ -Wno-logical-op-parentheses -Wno-parentheses -Oz WASM_EXTRA_LDFLAGS := -Wl,-z,stack-size=13312,--no-entry,--compress-relocations,--strip-all -WASM_EXTRA_LDFLAGS += -Wl,--export=__heap_base,--export=parseCJS,--export=sa +WASM_EXTRA_LDFLAGS += -Wl,--export=__heap_base,--export=parseCJS,--export=parseCJSGrouped,--export=sa WASM_EXTRA_LDFLAGS += -Wl,--export=e,--export=re,--export=es,--export=ee WASM_EXTRA_LDFLAGS += -Wl,--export=rre,--export=ree,--export=res,--export=ru,--export=us,--export=ue -lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c +lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c src/reexport-analysis.h @mkdir -p lib $(WASM_CC) $(WASM_CFLAGS) $(WASM_TARGET) $(WASM_EXTRA_CFLAGS) \ src/lexer.c -o lib/lexer.wasm \ diff --git a/include-wasm/cjs-module-lexer.h b/include-wasm/cjs-module-lexer.h index ab2adca..2e88907 100755 --- a/include-wasm/cjs-module-lexer.h +++ b/include-wasm/cjs-module-lexer.h @@ -72,6 +72,8 @@ uint32_t ee () { } // getReexportStart uint32_t res () { + // A null slice starts a group; Wasm exposes this sentinel as signed -1. + if (reexport_read_head->start == NULL) return UINT32_MAX; return reexport_read_head->start - source; } // getReexportEnd diff --git a/lexer.d.ts b/lexer.d.ts index cda07d8..c8f3318 100755 --- a/lexer.d.ts +++ b/lexer.d.ts @@ -1,8 +1,23 @@ export interface Exports { + exports: string[]; + /** Flat dependencies, or alternative lists when conditional final dependencies differ. */ + reexports: string[] | string[][]; + /** Present when competing replacements are analyzed. False for unsupported effects, filtered names, or limits. */ + complete?: boolean; +} + +export interface LegacyExports { exports: string[]; reexports: string[]; } +export interface ParseOptions { + baseline?: 'legacy' | 'flow-v1'; +} + +export declare function parse(source: string, name: string | undefined, options: { baseline: 'legacy' }): LegacyExports; +export declare function parse(source: string, name: string | undefined, options?: { baseline?: 'flow-v1' }): Exports; +export declare function parse(source: string, name: string | undefined, options?: ParseOptions): Exports | LegacyExports; export declare function parse(source: string, name?: string): Exports; export declare function init(): Promise; export declare function initSync(): void; diff --git a/lexer.js b/lexer.js index 2983aef..2cbfa61 100755 --- a/lexer.js +++ b/lexer.js @@ -13,17 +13,18 @@ let openTokenDepth, lastStarExportSpecifier, _exports, unsafeGetters, - reexports; + reexports, + moduleExportCount; function resetState () { openTokenDepth = 0; templateDepth = -1; lastTokenPos = -1; lastSlashWasDivision = false; - templateStack = new Array(1024); + if (templateStack === undefined) templateStack = new Array(1024); templateStackDepth = 0; - openTokenPosStack = new Array(1024); - openClassPosStack = new Array(1024); + if (openTokenPosStack === undefined) openTokenPosStack = new Array(1024); + if (openClassPosStack === undefined) openClassPosStack = new Array(1024); nextBraceIsClass = false; starExportMap = Object.create(null); lastStarExportSpecifier = null; @@ -31,6 +32,7 @@ function resetState () { _exports = new Set(); unsafeGetters = new Set(); reexports = new Set(); + moduleExportCount = 0; } // RequireType @@ -52,7 +54,13 @@ function pushTemplate () { templateDepth = ++openTokenDepth; } -function parseCJS (source, name = '@') { +/** @param {string} source @param {string} [name] @param {import('./lexer').ParseOptions} [options] */ +function parseCJS (source, name = '@', options) { + const baseline = options === undefined ? undefined : options.baseline; + if (baseline !== undefined && typeof baseline !== 'string') + throw new TypeError('Detection baseline must be a string'); + if (baseline !== undefined && baseline !== 'legacy' && baseline !== 'flow-v1') + throw new RangeError('Unknown detection baseline: ' + baseline); resetState(); try { parseSource(source); @@ -62,6 +70,10 @@ function parseCJS (source, name = '@') { e.loc = pos; throw e; } + if (baseline !== 'legacy' && moduleExportCount > 1) { + resetState(); + return require('./dist/lexer.js').parse(source, name, baseline === undefined ? undefined : { baseline }); + } const result = { exports: [..._exports].filter(expt => expt !== undefined && !unsafeGetters.has(expt)), reexports: [...reexports].filter(reexpt => reexpt !== undefined) }; resetState(); return result; @@ -897,6 +909,7 @@ function tryParseModuleExportsDotAssign () { pos = revertPos; } +/** @param {boolean} assign */ function tryParseExportsDotAssign (assign) { pos += 7; const revertPos = pos - 1; @@ -937,6 +950,8 @@ function tryParseExportsDotAssign (assign) { // module.exports = case 61/*=*/: { if (assign) { + if (moduleExportCount < 2 && source.charCodeAt(pos + 1) !== 61) + moduleExportCount++; if (reexports.size) reexports = new Set(); pos++; @@ -951,6 +966,13 @@ function tryParseExportsDotAssign (assign) { if (ch === 114/*r*/) tryParseRequire(ExportAssign); } + break; + } + case 38/*&*/: + case 124/*|*/: + case 63/*?*/: { + if (assign && moduleExportCount < 2 && source.charCodeAt(pos + 1) === ch && + source.charCodeAt(pos + 2) === 61) moduleExportCount++; } } pos = revertPos; @@ -1639,8 +1661,6 @@ function isExpressionTerminator (curPos) { return false; } -const initPromise = Promise.resolve(); - -module.exports.init = () => initPromise; -module.exports.initSync = () => {}; +module.exports.init = () => require('./dist/lexer.js').init(); +module.exports.initSync = () => require('./dist/lexer.js').initSync(); module.exports.parse = parseCJS; diff --git a/lib/lexer.wasm b/lib/lexer.wasm index 08fbd54..cbc6a44 100755 Binary files a/lib/lexer.wasm and b/lib/lexer.wasm differ diff --git a/src/lexer.c b/src/lexer.c index b6576d4..e7ed7f8 100755 --- a/src/lexer.c +++ b/src/lexer.c @@ -10,6 +10,7 @@ const uint16_t* EMPTY_CHAR = &__empty_char; #define MAX_STAR_EXPORTS 256 const uint16_t* source; + bool lastSlashWasDivision; uint16_t templateStackDepth; uint16_t openTokenDepth; @@ -32,6 +33,11 @@ uint16_t* lastReexportEnd; uint32_t parse_error; uint32_t error = 0; uint32_t sourceLen; +uint32_t moduleExportCount; + +#ifdef __wasm__ +#include "reexport-analysis.h" +#endif uint16_t templateStack_[STACK_DEPTH]; uint16_t* openTokenPosStack_[STACK_DEPTH]; @@ -72,6 +78,7 @@ static inline __attribute__((always_inline)) bool pushTemplate () { uint32_t parseCJS (uint16_t* _source, uint32_t _sourceLen, void (*_addExport)(const uint16_t*, const uint16_t*), void (*_addReexport)(const uint16_t*, const uint16_t*), void (*_addUnsafeGetter)(const uint16_t*, const uint16_t*), void (*_clearReexports)()) { source = _source; sourceLen = _sourceLen; + moduleExportCount = 0; if (_addExport) addExport = _addExport; if (_addReexport) @@ -263,7 +270,12 @@ uint32_t parseCJS (uint16_t* _source, uint32_t _sourceLen, void (*_addExport)(co return error; // success +#ifdef __wasm__ + // The low byte is reserved for parse errors in the host wrapper. + return moduleExportCount << 8; +#else return 0; +#endif } void tryBacktrackAddStarExportBinding (uint16_t* bPos) { @@ -939,6 +951,8 @@ void tryParseExportsDotAssign (bool assign) { // module.exports = case '=': { if (assign) { + if (moduleExportCount < 2 && pos[1] != '=') + moduleExportCount++; clearReexports(); pos++; ch = commentWhitespace(); @@ -952,6 +966,13 @@ void tryParseExportsDotAssign (bool assign) { if (ch == 'r') tryParseRequire(ExportAssign); } + break; + } + case '&': + case '|': + case '?': { + if (assign && moduleExportCount < 2 && end - pos >= 2 && pos[1] == ch && pos[2] == '=') + moduleExportCount++; } } pos = revertPos; diff --git a/src/lexer.js b/src/lexer.js index 9989658..8d4dfc1 100755 --- a/src/lexer.js +++ b/src/lexer.js @@ -3,7 +3,13 @@ let wasm; const isLE = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1; const hasBuffer = typeof Buffer !== 'undefined'; -export function parse (source, name = '@') { +/** @param {string} source @param {string} [name] @param {import('../lexer').ParseOptions} [options] */ +export function parse (source, name = '@', options) { + const baseline = options === undefined ? undefined : options.baseline; + if (baseline !== undefined && typeof baseline !== 'string') + throw new TypeError('Detection baseline must be a string'); + if (baseline !== undefined && baseline !== 'legacy' && baseline !== 'flow-v1') + throw new RangeError('Unknown detection baseline: ' + baseline); if (!wasm) initSync(); @@ -21,7 +27,9 @@ export function parse (source, name = '@') { else (isLE ? copyLE : copyBE)(source, new Uint16Array(wasm.memory.buffer, addr, len)); - const err_code = wasm.parseCJS(addr, source.length, 0, 0, 0); + const status = baseline === 'legacy' ? wasm.parseCJS(addr, source.length, 0, 0, 0) : + wasm.parseCJSGrouped(addr, source.length); + const err_code = status & 255; if (err_code) { const err = new Error(`Parse error ${name}${wasm.e()}:${source.slice(0, wasm.e()).split('\n').length}:${wasm.e() - source.lastIndexOf('\n', wasm.e() - 1)}`); @@ -32,21 +40,49 @@ export function parse (source, name = '@') { } let exports = new Set(), reexports = new Set(), unsafeGetters = new Set(); + const groups = status & 1024 ? [] : undefined; + let complete = !(status & 2048); while (wasm.rre()) { - const reexptStr = decode(source.slice(wasm.res(), wasm.ree())); + const start = wasm.res(); + if (start === -1) { + reexports = new Set(); + groups.push(reexports); + continue; + } + const reexptStr = decode(source.slice(start, wasm.ree())); if (reexptStr) reexports.add(reexptStr); + else if (reexptStr === undefined) + complete = false; } while (wasm.ru()) unsafeGetters.add(decode(source.slice(wasm.us(), wasm.ue()))); while (wasm.re()) { - let exptStr = decode(source.slice(wasm.es(), wasm.ee())); + const text = source.slice(wasm.es(), wasm.ee()); + let exptStr = status & 1024 && text.charCodeAt(0) >= 48 && text.charCodeAt(0) <= 57 ? + String(Number(text)) : decode(text); if (exptStr !== undefined && !unsafeGetters.has(exptStr)) exports.add(exptStr); + else if (exptStr === undefined) + complete = false; } - return { exports: [...exports], reexports: [...reexports] }; + if (!(status & 1024)) return { exports: [...exports], reexports: [...reexports] }; + if (groups.length === 1) + return { exports: [...exports], reexports: [...reexports], complete }; + const alternatives = []; + const seen = new Set(); + let hasDependencies = false; + for (const group of groups) { + const values = [...group]; + const key = JSON.stringify(values); + if (seen.has(key)) continue; + seen.add(key); + alternatives.push(values); + if (values.length) hasDependencies = true; + } + return { exports: [...exports], reexports: hasDependencies ? alternatives.length === 1 ? alternatives[0] : alternatives : [], complete }; } function decode (str) { diff --git a/src/reexport-analysis.h b/src/reexport-analysis.h new file mode 100644 index 0000000..93e72b7 --- /dev/null +++ b/src/reexport-analysis.h @@ -0,0 +1,763 @@ +#define FLOW_TOKENS 8192 +#define FLOW_DEPTH 64 +#define FLOW_OUTCOMES 32 +#define FLOW_STEPS 65536 +#define FLOW_ANALYZED 1024 +#define FLOW_INCOMPLETE 2048 + +enum FlowKind { + FT_IDENTIFIER = 256, FT_STRING, FT_NUMBER, FT_EQUAL, FT_NOT_EQUAL, + FT_AND, FT_OR, FT_NULLISH, FT_AND_ASSIGN, FT_OR_ASSIGN, FT_NULLISH_ASSIGN, FT_SPREAD, + FN_NUMBER, FN_STRING, FN_BOOLEAN, FN_NULL, FN_UNDEFINED, FN_IDENTIFIER, + FN_MEMBER, FN_CALL, FN_UNARY, FN_BINARY, FN_CONDITIONAL, FN_OBJECT, FN_PROPERTY, FN_SPREAD, + FN_EXPRESSION, FN_BLOCK, FN_IF, FN_EMPTY +}; + +enum FlowValueKind { + FV_UNKNOWN, FV_NUMBER, FV_STRING, FV_BOOLEAN, FV_NULL, FV_UNDEFINED, + FV_MODULE, FV_OBJECT, FV_OPAQUE_OBJECT, FV_FUNCTION, FV_UNKNOWN_STRING, + FV_TYPE_NUMBER, FV_TYPE_STRING, FV_TYPE_BOOLEAN, FV_TYPE_OBJECT, FV_TYPE_FUNCTION, FV_TYPE_UNDEFINED +}; + +typedef struct { + uint32_t start, end; + uint16_t kind; +} FlowToken; + +typedef struct { + uint16_t kind, op, left, right, third, next; + uint32_t start, end; + double number; +} FlowNode; + +typedef struct { + uint16_t kind, node; + int8_t truth; + double number; +} FlowValue; + +typedef struct { FlowValue root, value; } FlowEvaluation; +typedef struct { uint32_t count; FlowEvaluation values[FLOW_OUTCOMES]; } FlowResult; + +typedef struct { + FlowToken* tokens; + FlowNode* nodes; + FlowResult* results; + uint32_t tokenCount, nodeCount, cursor, depth, steps; + bool failed; +} FlowContext; + +static FlowContext flow; + +static bool flowStep () { + if (++flow.steps > FLOW_STEPS) flow.failed = true; + return !flow.failed; +} + +static bool flowReserve (uint32_t bytes) { + uint64_t required = (uint64_t)(uintptr_t)analysis_head + bytes; + if (required > UINT32_MAX) return false; + uint64_t capacity = (uint64_t)__builtin_wasm_memory_size(0) * 65536; + return required <= capacity || + __builtin_wasm_memory_grow(0, (required - capacity + 65535) / 65536) != (size_t)-1; +} + +static bool flowLineBreak (uint16_t ch) { + return ch == '\r' || ch == '\n' || ch == 0x2028 || ch == 0x2029; +} + +static bool flowSpace (uint16_t ch) { + return ch == 32 || ch >= 9 && ch <= 13 || ch == 0xA0 || ch == 0x1680 || + ch >= 0x2000 && ch <= 0x200A || ch == 0x2028 || ch == 0x2029 || + ch == 0x202F || ch == 0x205F || ch == 0x3000 || ch == 0xFEFF; +} + +static bool flowIdentifier (uint16_t ch) { + return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch == '_' || ch == '$' || + ch > 127 && !flowSpace(ch); +} + +static bool flowText (uint32_t start, uint32_t end, const char* text) { + while (start < end && *text && source[start] == (uint8_t)*text) { start++; text++; } + return start == end && !*text; +} + +static bool flowName (uint16_t node, const char* name) { + return flow.nodes[node].kind == FN_IDENTIFIER && + flowText(flow.nodes[node].start, flow.nodes[node].end, name); +} + +static int flowDigit (uint16_t ch) { + if (ch >= '0' && ch <= '9') return ch - '0'; + if (ch >= 'a' && ch <= 'f') return ch - 'a' + 10; + if (ch >= 'A' && ch <= 'F') return ch - 'A' + 10; + return -1; +} + +static int32_t flowStringNext (FlowNode* node, uint32_t* cursor, uint16_t* pending) { + if (*pending) { uint16_t ch = *pending; *pending = 0; return ch; } + while (*cursor < node->end - 1 && flowStep()) { + uint16_t ch = source[(*cursor)++]; + if (ch != '\\') return ch; + ch = source[(*cursor)++]; + if (flowLineBreak(ch)) { + if (ch == '\r' && source[*cursor] == '\n') (*cursor)++; + continue; + } + switch (ch) { + case 'b': return '\b'; + case 'f': return '\f'; + case 'n': return '\n'; + case 'r': return '\r'; + case 't': return '\t'; + case 'v': return '\v'; + case 'x': + case 'u': { + uint32_t code = 0, digits = ch == 'x' ? 2 : 4; + bool brace = ch == 'u' && source[*cursor] == '{'; + if (brace) { (*cursor)++; digits = 6; } + uint32_t read = 0; + while (read < digits && *cursor < node->end - 1) { + int digit = flowDigit(source[*cursor]); + if (digit < 0) break; + code = code * 16 + digit; + (*cursor)++; read++; + } + if (brace ? !read || source[(*cursor)++] != '}' : read != digits) { + flow.failed = true; return -1; + } + if (code > 0x10FFFF) { flow.failed = true; return -1; } + if (code > 0xFFFF) { + *pending = 0xDC00 + ((code - 0x10000) & 1023); + return 0xD800 + ((code - 0x10000) >> 10); + } + return code; + } + default: { + if (ch >= '0' && ch <= '7') { + uint32_t code = ch - '0', count = ch <= '3' ? 2 : 1; + while (count-- && source[*cursor] >= '0' && source[*cursor] <= '7') + code = code * 8 + source[(*cursor)++] - '0'; + return code; + } + return ch; + } + } + } + return -1; +} + +static bool flowStringText (uint16_t index, const char* text) { + FlowNode* node = &flow.nodes[index]; + uint32_t cursor = node->start + 1; + uint16_t pending = 0; + int32_t ch; + while ((ch = flowStringNext(node, &cursor, &pending)) >= 0) { + if (!*text || ch != (uint8_t)*text++) return false; + } + return !*text; +} + +static double flowNumber (FlowToken token) { + uint32_t cursor = token.start, radix = 10, fraction = 0; + int exponent = 0; + uint64_t value = 0; + bool point = false; + if (token.end - cursor > 128) { flow.failed = true; return 0; } + if (source[cursor] == '0' && cursor + 1 < token.end) { + uint16_t prefix = source[cursor + 1] | 32; + if (prefix == 'x' || prefix == 'b' || prefix == 'o') { + radix = prefix == 'x' ? 16 : prefix == 'b' ? 2 : 8; + cursor += 2; + } else if (source[cursor + 1] >= '0' && source[cursor + 1] <= '9') { + flow.failed = true; return 0; + } + } + while (cursor < token.end) { + uint16_t ch = source[cursor++]; + if (radix == 10 && ch == '.') { point = true; continue; } + if (radix == 10 && (ch == 'e' || ch == 'E')) { + bool negative = source[cursor] == '-'; + if (negative || source[cursor] == '+') cursor++; + while (cursor < token.end && exponent <= 128) exponent = exponent * 10 + source[cursor++] - '0'; + if (negative) exponent = -exponent; + break; + } + int digit = flowDigit(ch); + if (digit < 0 || digit >= radix || value > (9007199254740991ULL - digit) / radix) { + flow.failed = true; return 0; + } + value = value * radix + digit; + if (point) fraction++; + } + if (!value) return 0; + exponent -= fraction; + if (exponent < -15 || exponent > 15) { flow.failed = true; return 0; } + uint64_t scale = 1; + for (uint32_t i = 0; i < (uint32_t)(exponent < 0 ? -exponent : exponent); i++) scale *= 10; + if (exponent > 0 && value > 9007199254740991ULL / scale) { flow.failed = true; return 0; } + return exponent > 0 ? (double)(value * scale) : (double)value / scale; +} + +static void flowTokenize () { + uint32_t cursor = 0; + if (sourceLen > 1 && source[0] == '#' && source[1] == '!') { + while (cursor < sourceLen && !flowLineBreak(source[cursor]) && flowStep()) cursor++; + } + while (cursor < sourceLen && flowStep()) { + uint32_t start = cursor; + uint16_t ch = source[cursor++], kind = ch; + if (flowSpace(ch)) continue; + if (ch == '/' && cursor < sourceLen && source[cursor] == '/') { + while (cursor < sourceLen && !flowLineBreak(source[cursor]) && flowStep()) cursor++; + continue; + } + if (ch == '/' && cursor < sourceLen && source[cursor] == '*') { + cursor++; + while (cursor + 1 < sourceLen && !(source[cursor] == '*' && source[cursor + 1] == '/') && flowStep()) + cursor++; + if (cursor + 1 >= sourceLen) { flow.failed = true; return; } + cursor += 2; continue; + } + if (ch == '\'' || ch == '"') { + kind = FT_STRING; + while (cursor < sourceLen && source[cursor] != ch && flowStep()) { + if (source[cursor] == '\\') cursor++; + cursor++; + } + if (cursor >= sourceLen) { flow.failed = true; return; } + cursor++; + } else if (flowIdentifier(ch)) { + kind = FT_IDENTIFIER; + while (cursor < sourceLen && (flowIdentifier(source[cursor]) || + source[cursor] >= '0' && source[cursor] <= '9') && flowStep()) cursor++; + } else if (ch >= '0' && ch <= '9') { + kind = FT_NUMBER; + bool radix = ch == '0' && cursor < sourceLen && + ((source[cursor] | 32) == 'x' || (source[cursor] | 32) == 'b' || (source[cursor] | 32) == 'o'); + if (radix) cursor++; + while (cursor < sourceLen && flowStep()) { + ch = source[cursor]; + if (radix ? flowDigit(ch) >= 0 : ch >= '0' && ch <= '9' || ch == '.') { cursor++; continue; } + if (!radix && (ch == 'e' || ch == 'E')) { + cursor++; + if (source[cursor] == '+' || source[cursor] == '-') cursor++; + while (cursor < sourceLen && source[cursor] >= '0' && source[cursor] <= '9') cursor++; + } + break; + } + } else if ((ch == '+' || ch == '-') && cursor < sourceLen && source[cursor] == ch) { + flow.failed = true; return; + } else if ((ch == '=' || ch == '!') && cursor + 1 < sourceLen && + source[cursor] == '=' && source[cursor + 1] == '=') { + kind = ch == '=' ? FT_EQUAL : FT_NOT_EQUAL; cursor += 2; + } else if ((ch == '&' || ch == '|' || ch == '?') && cursor < sourceLen && source[cursor] == ch) { + cursor++; + bool assign = cursor < sourceLen && source[cursor] == '='; + kind = ch == '&' ? (assign ? FT_AND_ASSIGN : FT_AND) : + ch == '|' ? (assign ? FT_OR_ASSIGN : FT_OR) : (assign ? FT_NULLISH_ASSIGN : FT_NULLISH); + if (assign) cursor++; + } else if (ch == '.' && cursor + 1 < sourceLen && source[cursor] == '.' && source[cursor + 1] == '.') { + kind = FT_SPREAD; cursor += 2; + } + if (flow.tokenCount == FLOW_TOKENS) { flow.failed = true; return; } + flow.tokens[flow.tokenCount++] = (FlowToken){ start, cursor, kind }; + } +} + +static uint16_t flowPeek () { + return flow.cursor < flow.tokenCount ? flow.tokens[flow.cursor].kind : 0; +} + +static bool flowTake (uint16_t kind) { + if (flowPeek() != kind) return false; + flow.cursor++; return true; +} + +static void flowExpect (uint16_t kind) { + if (!flowTake(kind)) flow.failed = true; +} + +static uint16_t flowNode (uint16_t kind, uint16_t left, uint16_t right, FlowToken token) { + if (flow.nodeCount == FLOW_TOKENS || !flowStep()) { flow.failed = true; return 0; } + uint16_t index = ++flow.nodeCount; + FlowNode* node = &flow.nodes[index]; + node->kind = kind; node->op = 0; + node->left = left; node->right = right; node->third = 0; node->next = 0; + node->start = token.start; node->end = token.end; node->number = 0; + return index; +} + +static uint16_t flowLiteral (FlowToken token) { + uint16_t kind = token.kind == FT_STRING ? FN_STRING : token.kind == FT_NUMBER ? FN_NUMBER : FN_IDENTIFIER; + if (token.kind == FT_IDENTIFIER) { + if (flowText(token.start, token.end, "true") || flowText(token.start, token.end, "false")) kind = FN_BOOLEAN; + else if (flowText(token.start, token.end, "null")) kind = FN_NULL; + else if (flowText(token.start, token.end, "undefined")) kind = FN_UNDEFINED; + } + uint16_t node = flowNode(kind, 0, 0, token); + if (kind == FN_NUMBER) flow.nodes[node].number = flowNumber(token); + if (kind == FN_BOOLEAN) flow.nodes[node].number = source[token.start] == 't'; + return node; +} + +static uint32_t flowPower (uint16_t kind) { + switch (kind) { + case ',': return 1; + case '=': case FT_AND_ASSIGN: case FT_OR_ASSIGN: case FT_NULLISH_ASSIGN: return 2; + case '?': return 3; + case FT_OR: case FT_NULLISH: return 4; + case FT_AND: return 5; + case FT_EQUAL: case FT_NOT_EQUAL: return 6; + default: return 0; + } +} + +static uint16_t flowExpression (uint32_t minimum); + +static uint16_t flowObject (FlowToken token) { + uint16_t object = flowNode(FN_OBJECT, 0, 0, token), tail = 0; + while (flowPeek() != '}' && !flow.failed) { + if (!flowPeek()) { flow.failed = true; break; } + uint16_t entry, key = 0; + if (flowTake(FT_SPREAD)) { + entry = flowNode(FN_SPREAD, flowExpression(2), 0, token); + } else { + FlowToken name = flow.tokens[flow.cursor++]; + if (name.kind != FT_IDENTIFIER && name.kind != FT_STRING && name.kind != FT_NUMBER) { + flow.failed = true; break; + } + key = name.kind == FT_IDENTIFIER ? flowNode(FN_IDENTIFIER, 0, 0, name) : flowLiteral(name); + uint16_t value; + if (flowTake(':')) { + if (flowName(key, "__proto__") || name.kind == FT_STRING && flowStringText(key, "__proto__")) + flow.failed = true; + value = flowExpression(2); + } else { + if (name.kind != FT_IDENTIFIER) flow.failed = true; + value = flowLiteral(name); + } + entry = flowNode(FN_PROPERTY, key, value, name); + } + if (tail) flow.nodes[tail].next = entry; + else flow.nodes[object].left = entry; + tail = entry; + if (!flowTake(',')) break; + } + flowExpect('}'); + return object; +} + +static uint16_t flowExpression (uint32_t minimum) { + if (++flow.depth > FLOW_DEPTH || !flowPeek() || !flowStep()) { flow.failed = true; flow.depth--; return 0; } + FlowToken token = flow.tokens[flow.cursor++]; + uint16_t left; + if (token.kind == '(') { + left = flowExpression(1); flowExpect(')'); + } else if (token.kind == '{') left = flowObject(token); + else if (token.kind == '!' || token.kind == '+' || token.kind == '-' || + token.kind == FT_IDENTIFIER && (flowText(token.start, token.end, "void") || + flowText(token.start, token.end, "typeof"))) { + left = flowNode(FN_UNARY, flowExpression(12), 0, token); + flow.nodes[left].op = token.kind == FT_IDENTIFIER ? (source[token.start] == 'v' ? 'v' : 't') : token.kind; + } else if (token.kind == FT_IDENTIFIER || token.kind == FT_STRING || token.kind == FT_NUMBER) + left = flowLiteral(token); + else { flow.failed = true; left = 0; } + while (!flow.failed && flowStep()) { + if (flowPeek() == '.' || flowPeek() == '[') { + uint16_t op = flow.tokens[flow.cursor++].kind, key; + if (op == '.') { + if (flowPeek() != FT_IDENTIFIER) { flow.failed = true; break; } + key = flowNode(FN_IDENTIFIER, 0, 0, flow.tokens[flow.cursor++]); + } else { key = flowExpression(1); flowExpect(']'); } + left = flowNode(FN_MEMBER, left, key, token); flow.nodes[left].op = op; + } else if (flowTake('(')) { + uint16_t call = flowNode(FN_CALL, left, 0, token), tail = 0; + while (flowPeek() != ')' && !flow.failed) { + uint16_t argument = flowExpression(2); + if (tail) flow.nodes[tail].next = argument; + else flow.nodes[call].right = argument; + tail = argument; + if (!flowTake(',')) break; + } + flowExpect(')'); left = call; + } else { + uint32_t power = flowPower(flowPeek()); + if (!power || power < minimum) break; + FlowToken operator = flow.tokens[flow.cursor++]; + if (operator.kind == '?') { + uint16_t consequent = flowExpression(2); + flowExpect(':'); + uint16_t alternate = flowExpression(2); + left = flowNode(FN_CONDITIONAL, left, consequent, operator); flow.nodes[left].third = alternate; + } else { + uint16_t right = flowExpression(power == 2 ? power : power + 1); + left = flowNode(FN_BINARY, left, right, operator); flow.nodes[left].op = operator.kind; + } + } + } + flow.depth--; return left; +} + +static uint16_t flowStatement () { + if (++flow.depth > FLOW_DEPTH || !flowPeek() || !flowStep()) { flow.failed = true; flow.depth--; return 0; } + FlowToken token = flow.tokens[flow.cursor++]; + if (token.kind == FT_IDENTIFIER) { + static const char* const unsupported[] = { + "return", "throw", "break", "continue", "function", "class", "var", "let", "const", + "for", "while", "do", "switch", "case", "try", "catch", "finally", "with", "debugger", "in", "instanceof" + }; + for (uint32_t i = 0; i < sizeof(unsupported) / sizeof(unsupported[0]); i++) + if (flowText(token.start, token.end, unsupported[i])) { flow.failed = true; flow.depth--; return 0; } + } + uint16_t node; + if (token.kind == ';') node = flowNode(FN_EMPTY, 0, 0, token); + else if (token.kind == '{') { + node = flowNode(FN_BLOCK, 0, 0, token); + uint16_t tail = 0; + while (flowPeek() != '}' && !flow.failed) { + uint16_t statement = flowStatement(); + if (tail) flow.nodes[tail].next = statement; + else flow.nodes[node].left = statement; + tail = statement; + } + flowExpect('}'); + } else if (token.kind == FT_IDENTIFIER && flowText(token.start, token.end, "if")) { + flowExpect('('); uint16_t condition = flowExpression(1); flowExpect(')'); + node = flowNode(FN_IF, condition, flowStatement(), token); + if (flowPeek() == FT_IDENTIFIER && + flowText(flow.tokens[flow.cursor].start, flow.tokens[flow.cursor].end, "else")) { + flow.cursor++; flow.nodes[node].third = flowStatement(); + } + } else { + flow.cursor--; + node = flowNode(FN_EXPRESSION, flowExpression(1), 0, token); + if (!flowTake(';') && flowPeek() && flowPeek() != '}') { + uint32_t start = flow.tokens[flow.cursor - 1].end, next = flow.tokens[flow.cursor].start; + while (start < next && !flowLineBreak(source[start])) start++; + if (start == next || flowPeek() == '+' || flowPeek() == '-') flow.failed = true; + } + } + flow.depth--; return node; +} + +static FlowValue flowValue (uint16_t kind, uint16_t node, int8_t truth, double number) { + return (FlowValue){ kind, node, truth, number }; +} + +static bool flowSame (FlowValue first, FlowValue second) { + return first.kind == second.kind && first.node == second.node && + first.truth == second.truth && first.number == second.number; +} + +static void flowAppend (FlowResult* result, FlowValue root, FlowValue value) { + for (uint32_t i = 0; i < result->count && flowStep(); i++) + if (flowSame(result->values[i].root, root) && flowSame(result->values[i].value, value)) return; + if (result->count == FLOW_OUTCOMES) { flow.failed = true; return; } + result->values[result->count++] = (FlowEvaluation){ root, value }; +} + +static void flowMerge (FlowResult* target, FlowResult* values) { + for (uint32_t i = 0; i < values->count && flowStep(); i++) + flowAppend(target, values->values[i].root, values->values[i].value); +} + +static bool flowRoot (uint16_t index) { + FlowNode* node = &flow.nodes[index]; + return node->kind == FN_MEMBER && flowName(node->left, "module") && + (node->op == '.' ? flowName(node->right, "exports") : + flow.nodes[node->right].kind == FN_STRING && flowStringText(node->right, "exports")); +} + +static bool flowRequire (uint16_t index) { + FlowNode* node = &flow.nodes[index]; + return node->kind == FN_CALL && flowName(node->left, "require") && node->right && + !flow.nodes[node->right].next && flow.nodes[node->right].kind == FN_STRING; +} + +static bool flowStringsEqual (uint16_t first, uint16_t second) { + uint32_t firstCursor = flow.nodes[first].start + 1, secondCursor = flow.nodes[second].start + 1; + uint16_t firstPending = 0, secondPending = 0; + int32_t ch; + do { + ch = flowStringNext(&flow.nodes[first], &firstCursor, &firstPending); + if (ch != flowStringNext(&flow.nodes[second], &secondCursor, &secondPending)) return false; + } while (ch >= 0 && !flow.failed); + return true; +} + +static const char* flowTypeText (uint16_t kind) { + switch (kind) { + case FV_TYPE_NUMBER: return "number"; + case FV_TYPE_STRING: return "string"; + case FV_TYPE_BOOLEAN: return "boolean"; + case FV_TYPE_OBJECT: return "object"; + case FV_TYPE_FUNCTION: return "function"; + case FV_TYPE_UNDEFINED: return "undefined"; + default: return NULL; + } +} + +static int8_t flowEqual (FlowValue first, FlowValue second) { + if (first.kind == FV_UNKNOWN || second.kind == FV_UNKNOWN || + first.kind == FV_MODULE || second.kind == FV_MODULE || + first.kind == FV_UNKNOWN_STRING || second.kind == FV_UNKNOWN_STRING) return -1; + if ((first.kind == FV_OPAQUE_OBJECT && (second.kind == FV_OBJECT || second.kind == FV_OPAQUE_OBJECT)) || + (second.kind == FV_OPAQUE_OBJECT && first.kind == FV_OBJECT)) return -1; + const char* firstType = flowTypeText(first.kind), *secondType = flowTypeText(second.kind); + if (firstType || secondType) { + if (firstType && secondType) return first.kind == second.kind; + FlowValue string = firstType ? second : first; + return string.kind == FV_STRING && flowStringText(string.node, firstType ? firstType : secondType); + } + if (first.kind != second.kind) return 0; + if (first.kind == FV_STRING) return flowStringsEqual(first.node, second.node); + if (first.kind == FV_NUMBER || first.kind == FV_BOOLEAN) { + if (first.kind == FV_BOOLEAN && (first.truth < 0 || second.truth < 0)) return -1; + return first.number == second.number; + } + if (first.kind == FV_OBJECT) return first.node == second.node; + if (first.kind == FV_FUNCTION) return 1; + if (first.kind == FV_OPAQUE_OBJECT) return -1; + return 1; +} + +static FlowValue flowTypeof (FlowValue value) { + uint16_t kind; + switch (value.kind) { + case FV_NUMBER: kind = FV_TYPE_NUMBER; break; + case FV_BOOLEAN: kind = FV_TYPE_BOOLEAN; break; + case FV_NULL: case FV_OBJECT: case FV_OPAQUE_OBJECT: kind = FV_TYPE_OBJECT; break; + case FV_FUNCTION: kind = FV_TYPE_FUNCTION; break; + case FV_UNDEFINED: kind = FV_TYPE_UNDEFINED; break; + case FV_UNKNOWN: case FV_MODULE: return flowValue(FV_UNKNOWN_STRING, 0, 1, 0); + default: kind = FV_TYPE_STRING; + } + return flowValue(kind, 0, 1, 0); +} + +static int8_t flowRoute (uint16_t op, FlowValue value) { + if (op == FT_NULLISH || op == FT_NULLISH_ASSIGN) + return value.kind == FV_UNKNOWN || value.kind == FV_MODULE ? -1 : + value.kind == FV_NULL || value.kind == FV_UNDEFINED; + return value.truth < 0 ? -1 : (op == FT_OR || op == FT_OR_ASSIGN ? !value.truth : value.truth); +} + +static void flowEvaluate (uint16_t index, FlowValue root, FlowResult* result, uint32_t depth); + +static void flowEvaluate (uint16_t index, FlowValue root, FlowResult* result, uint32_t depth) { + result->count = 0; + if (depth == FLOW_DEPTH || !flowStep()) { flow.failed = true; return; } + FlowNode* node = &flow.nodes[index]; + FlowResult* first = &flow.results[depth * 3], *second = first + 1; + FlowValue unknown = flowValue(FV_UNKNOWN, 0, -1, 0); + switch (node->kind) { + case FN_NUMBER: flowAppend(result, root, flowValue(FV_NUMBER, 0, node->number != 0, node->number)); return; + case FN_BOOLEAN: flowAppend(result, root, flowValue(FV_BOOLEAN, 0, node->number != 0, node->number)); return; + case FN_NULL: flowAppend(result, root, flowValue(FV_NULL, 0, 0, 0)); return; + case FN_UNDEFINED: flowAppend(result, root, flowValue(FV_UNDEFINED, 0, 0, 0)); return; + case FN_STRING: { + uint32_t cursor = node->start + 1; uint16_t pending = 0; + flowAppend(result, root, flowValue(FV_STRING, index, flowStringNext(node, &cursor, &pending) >= 0, 0)); return; + } + case FN_IDENTIFIER: { + if (flowName(index, "require")) flowAppend(result, root, flowValue(FV_FUNCTION, 0, 1, 0)); + else if (flowName(index, "module") || flowName(index, "exports")) + flowAppend(result, root, flowValue(FV_OPAQUE_OBJECT, 0, 1, 0)); + else flowAppend(result, unknown, unknown); + return; + } + case FN_MEMBER: + if (flowRoot(index)) flowAppend(result, root, root); + else flowAppend(result, unknown, unknown); + return; + case FN_CALL: + if (flowRequire(index)) flowAppend(result, unknown, flowValue(FV_MODULE, index, -1, 0)); + else { flow.failed = true; } + return; + case FN_OBJECT: { + first->count = 0; flowAppend(first, root, root); + for (uint16_t entry = node->left; entry && !flow.failed; entry = flow.nodes[entry].next) { + FlowNode* property = &flow.nodes[entry]; + uint16_t expression = property->kind == FN_SPREAD ? property->left : property->right; + if (property->kind == FN_SPREAD && !flowRequire(expression) && + flow.nodes[expression].kind != FN_OBJECT && + flow.nodes[expression].kind != FN_NULL && flow.nodes[expression].kind != FN_UNDEFINED) { + flow.failed = true; break; + } + second->count = 0; + for (uint32_t i = 0; i < first->count && !flow.failed; i++) { + FlowResult* child = &flow.results[depth * 3 + 2]; + flowEvaluate(expression, first->values[i].root, child, depth + 1); + for (uint32_t j = 0; j < child->count && !flow.failed; j++) + flowAppend(second, child->values[j].root, child->values[j].root); + } + FlowResult* swap = first; first = second; second = swap; + } + for (uint32_t i = 0; i < first->count && !flow.failed; i++) + flowAppend(result, first->values[i].root, flowValue(FV_OBJECT, index, 1, 0)); + return; + } + case FN_UNARY: + flowEvaluate(node->left, root, first, depth + 1); + for (uint32_t i = 0; i < first->count && !flow.failed; i++) { + FlowEvaluation item = first->values[i]; FlowValue value; + if (node->op == '!') value = flowValue(FV_BOOLEAN, 0, + item.value.truth < 0 ? -1 : !item.value.truth, item.value.truth < 0 ? 0 : !item.value.truth); + else if (node->op == 'v') value = flowValue(FV_UNDEFINED, 0, 0, 0); + else if (node->op == 't') value = flowTypeof(item.value); + else if (item.value.kind == FV_NUMBER) value = flowValue(FV_NUMBER, 0, + item.value.number != 0, node->op == '-' ? -item.value.number : item.value.number); + else { flow.failed = true; break; } + flowAppend(result, item.root, value); + } + return; + case FN_CONDITIONAL: + flowEvaluate(node->left, root, first, depth + 1); + for (uint32_t i = 0; i < first->count && !flow.failed; i++) { + FlowEvaluation item = first->values[i]; + if (item.value.truth != 0) { + flowEvaluate(node->right, item.root, second, depth + 1); flowMerge(result, second); + } + if (item.value.truth != 1) { + flowEvaluate(node->third, item.root, second, depth + 1); flowMerge(result, second); + } + } + return; + case FN_BINARY: { + bool assign = node->op == '=' || node->op == FT_AND_ASSIGN || + node->op == FT_OR_ASSIGN || node->op == FT_NULLISH_ASSIGN; + if (assign && !flowRoot(node->left)) { flow.failed = true; return; } + if (node->op == '=') { + flowEvaluate(node->right, root, first, depth + 1); + for (uint32_t i = 0; i < first->count && !flow.failed; i++) + flowAppend(result, first->values[i].value, first->values[i].value); + return; + } + if (assign) { first->count = 0; flowAppend(first, root, root); } + else flowEvaluate(node->left, root, first, depth + 1); + for (uint32_t i = 0; i < first->count && !flow.failed; i++) { + FlowEvaluation item = first->values[i]; + if (node->op == FT_AND || node->op == FT_OR || node->op == FT_NULLISH || assign) { + int8_t route = flowRoute(node->op, item.value); + if (route != 1) flowAppend(result, item.root, item.value); + if (route != 0) { + flowEvaluate(node->right, item.root, second, depth + 1); + for (uint32_t j = 0; j < second->count && !flow.failed; j++) + flowAppend(result, assign ? second->values[j].value : second->values[j].root, second->values[j].value); + } + } else { + flowEvaluate(node->right, item.root, second, depth + 1); + for (uint32_t j = 0; j < second->count && !flow.failed; j++) { + FlowEvaluation right = second->values[j]; + if (node->op == ',') flowAppend(result, right.root, right.value); + else { + int8_t equal = flowEqual(item.value, right.value); + if (equal >= 0 && node->op == FT_NOT_EQUAL) equal = !equal; + flowAppend(result, right.root, flowValue(FV_BOOLEAN, 0, equal, equal < 0 ? 0 : equal)); + } + } + } + } + return; + } + default: flow.failed = true; + } +} + +static void flowExecute (uint16_t index, FlowValue root, FlowResult* result, uint32_t depth) { + result->count = 0; + if (depth == FLOW_DEPTH || !flowStep()) { flow.failed = true; return; } + FlowNode* node = &flow.nodes[index]; + FlowResult* first = &flow.results[depth * 3], *second = first + 1, *child = first + 2; + if (node->kind == FN_EMPTY) { flowAppend(result, root, root); return; } + if (node->kind == FN_EXPRESSION) { + flowEvaluate(node->left, root, first, depth + 1); + for (uint32_t i = 0; i < first->count && !flow.failed; i++) + flowAppend(result, first->values[i].root, first->values[i].root); + } else if (node->kind == FN_BLOCK) { + first->count = 0; flowAppend(first, root, root); + for (uint16_t statement = node->left; statement && !flow.failed; statement = flow.nodes[statement].next) { + second->count = 0; + for (uint32_t i = 0; i < first->count && !flow.failed; i++) { + flowExecute(statement, first->values[i].root, child, depth + 1); flowMerge(second, child); + } + FlowResult* swap = first; first = second; second = swap; + } + flowMerge(result, first); + } else if (node->kind == FN_IF) { + flowEvaluate(node->left, root, first, depth + 1); + for (uint32_t i = 0; i < first->count && !flow.failed; i++) { + FlowEvaluation item = first->values[i]; + if (item.value.truth != 0) { + flowExecute(node->right, item.root, second, depth + 1); flowMerge(result, second); + } + if (item.value.truth != 1) { + if (node->third) { + flowExecute(node->third, item.root, second, depth + 1); flowMerge(result, second); + } else flowAppend(result, item.root, item.root); + } + } + } else flow.failed = true; +} + +static void flowEmitObject (uint16_t index, uint32_t depth) { + if (depth == FLOW_DEPTH) { flow.failed = true; return; } + for (uint16_t entry = flow.nodes[index].left; entry && flowStep(); entry = flow.nodes[entry].next) { + FlowNode* property = &flow.nodes[entry], *value = &flow.nodes[property->left]; + if (property->kind == FN_PROPERTY) _addExport(source + value->start, source + value->end); + else if (flowRequire(property->left)) { + FlowNode* specifier = &flow.nodes[value->right]; + _addReexport(source + specifier->start, source + specifier->end); + } else if (value->kind == FN_OBJECT) flowEmitObject(property->left, depth + 1); + } +} + +static void flowClear () { + first_export = export_read_head = export_write_head = NULL; + first_reexport = reexport_read_head = reexport_write_head = NULL; + first_unsafe_getter = unsafe_getter_read_head = unsafe_getter_write_head = NULL; +} + +uint32_t parseCJSGrouped (uint16_t* input, uint32_t length) { + uint32_t status = parseCJS(input, length, NULL, NULL, NULL, NULL); + if ((status & 255) || moduleExportCount < 2) return status; + flowClear(); + flow.tokenCount = 0; flow.nodeCount = 0; flow.cursor = 0; flow.depth = 0; flow.steps = 0; flow.failed = false; + uint64_t aligned = ((uint64_t)(uintptr_t)analysis_head + 7) & ~7ULL; + uint32_t bytes = sizeof(FlowToken) * FLOW_TOKENS + sizeof(FlowNode) * (FLOW_TOKENS + 1) + + sizeof(FlowResult) * (FLOW_DEPTH * 3 + 1); + if (aligned > UINT32_MAX || aligned - (uintptr_t)analysis_head > UINT32_MAX - bytes || + !flowReserve(bytes + aligned - (uintptr_t)analysis_head)) return status | FLOW_ANALYZED | FLOW_INCOMPLETE; + flow.tokens = (FlowToken*)(uintptr_t)aligned; + flow.nodes = (FlowNode*)(flow.tokens + FLOW_TOKENS); + flow.results = (FlowResult*)(flow.nodes + FLOW_TOKENS + 1); + analysis_head = (void*)(flow.results + FLOW_DEPTH * 3 + 1); + flow.nodes[0].kind = FN_OBJECT; flow.nodes[0].left = 0; + flowTokenize(); + uint16_t block = flowNode(FN_BLOCK, 0, 0, (FlowToken){0}), tail = 0; + while (flowPeek() && !flow.failed) { + uint16_t statement = flowStatement(); + if (tail) flow.nodes[tail].next = statement; + else flow.nodes[block].left = statement; + tail = statement; + } + FlowResult* result = &flow.results[FLOW_DEPTH * 3]; + if (!flow.failed) flowExecute(block, flowValue(FV_OBJECT, 0, 1, 0), result, 0); + if (flow.failed || !flowReserve((flow.nodeCount + FLOW_OUTCOMES) * sizeof(Slice))) + return status | FLOW_ANALYZED | FLOW_INCOMPLETE; + bool complete = true; + for (uint32_t i = 0; i < result->count && flowStep(); i++) { + FlowValue value = result->values[i].root; + if (value.kind == FV_UNKNOWN || value.kind == FV_OPAQUE_OBJECT || value.kind == FV_FUNCTION) { + complete = false; continue; + } + _addReexport(NULL, NULL); + if (value.kind == FV_MODULE) { + FlowNode* specifier = &flow.nodes[flow.nodes[value.node].right]; + _addReexport(source + specifier->start, source + specifier->end); + } else if (value.kind == FV_OBJECT) flowEmitObject(value.node, 0); + } + if (flow.failed) { flowClear(); complete = false; } + return status | FLOW_ANALYZED | (complete ? 0 : FLOW_INCOMPLETE); +} diff --git a/test/_unit.js b/test/_unit.js index 59e5e6c..d42fa3c 100755 --- a/test/_unit.js +++ b/test/_unit.js @@ -15,9 +15,12 @@ async function loadParser () { else { parse = require('../lexer.js').parse; } + const parseSource = parse; + parse = /** @param {string} source @param {string} [name] */ (source, name) => + parseSource(source, name, { baseline: 'legacy' }); } -suite('Lexer', () => { +suite('Legacy lexer', () => { suiteSetup(async () => await loadParser()); test('automatic Wasm initialization', async () => { diff --git a/test/grouped.js b/test/grouped.js new file mode 100644 index 0000000..6483258 --- /dev/null +++ b/test/grouped.js @@ -0,0 +1,274 @@ +const assert = require('node:assert').strict; +const { mkdtempSync, rmSync, writeFileSync } = require('node:fs'); +const { createRequire } = require('node:module'); +const { tmpdir } = require('node:os'); +const { join } = require('node:path'); + +let parse; + +suite('Grouped reexports', () => { + suiteSetup(async () => { + if (process.env.WASM) { + const lexer = await import('../dist/lexer.mjs'); + await lexer.init(); + parse = lexer.parse; + } else if (process.env.WASM_SYNC) parse = require('../dist/lexer.js').parse; + else parse = require('../lexer.js').parse; + }); + + test('keeps flat results unless final dependency lists differ', () => { + for (const source of ['', 'exports.name = 1;', "module.exports = require('./one');", + "module.exports = {...require('./a'), ...require('./b')};"]) { + assert.deepStrictEqual(parse(source), parse(source, '@', { baseline: 'legacy' })); + } + assert.deepStrictEqual(parse("module.exports = require('./a'); module.exports = require('./b');").reexports, + ['./b']); + }); + test('deduplicates identical alternatives before choosing the result shape', () => { + assert.deepStrictEqual(parse("if (flag) module.exports = require('./same'); " + + 'else module.exports = require("./same");').reexports, ['./same']); + assert.deepStrictEqual(parse("if (flag) module.exports = {...require('./a'), ...require('./b'), ...require('./a')}; " + + "else module.exports = {...require('./a'), ...require('./b')};").reexports, ['./a', './b']); + assert.deepStrictEqual(parse("if (flag) module.exports = {}; else module.exports = require('./optional');").reexports, + [[], ['./optional']]); + assert.deepStrictEqual(parse('if (flag) module.exports = { first: 1 }; else module.exports = { second: 1 };').reexports, + []); + }); + test('groups conditional alternatives and drops overwritten dependencies by default', () => { + assert.deepStrictEqual(parse("if (flag) module.exports = require('./a'); else module.exports = require('./b');"), + { exports: [], reexports: [['./a'], ['./b']], complete: true }); + assert.deepStrictEqual(parse("module.exports = require('./a'); module.exports = require('./b');"), + { exports: [], reexports: ['./b'], complete: true }); + }); + test('groups dependencies combined in a final object', () => { + const source = "if (flag) module.exports = {...require('./a'), ...require('./b')}; " + + "else module.exports = {...require('./c'), ...require('./d')};"; + assert.deepStrictEqual(parse(source), { exports: [], reexports: [['./a', './b'], ['./c', './d']], complete: true }); + assert.deepStrictEqual(parse(source.replace('flag', 'true')), + { exports: [], reexports: ['./a', './b'], complete: true }); + }); + test('keeps the count gate and the explicit legacy baseline', () => { + for (const source of ['', 'exports.name = 1;', "module.exports = require('./one');"]) { + const result = parse(source); + assert.equal(Object.prototype.hasOwnProperty.call(result, 'complete'), false); + assert.equal(Object.prototype.hasOwnProperty.call(result, 'analysis'), false); + } + assert.deepStrictEqual(parse("module.exports = {...require('./a'), ...require('./b')};").reexports, + ['./a', './b']); + for (const extra of ['// module.exports = 2;', '/* module.exports = 2; */', "'module.exports = 2;'", + 'module.exports.name = 2;', 'module.exports === 2;', 'module.exports == 2;']) { + assert.equal(Object.prototype.hasOwnProperty.call(parse('module.exports = {}; ' + extra), 'complete'), false); + } + const source = "if (true) module.exports = require('./a'); else module.exports = require('./b');"; + assert.deepStrictEqual(parse(source, '@', { baseline: 'legacy' }), { exports: [], reexports: ['./b'] }); + assert.deepStrictEqual(parse(source, '@', { baseline: 'flow-v1' }), parse(source)); + }); + test('reads the baseline once and returns plain snapshot values', () => { + const source = "module.exports = require('./old'); module.exports = require('./final');"; + let reads = 0; + const result = parse(source, '@', { get baseline () { reads++; return 'flow-v1'; } }); + assert.equal(reads, 1); + assert.equal(Object.prototype.hasOwnProperty.call(result, 'analysis'), false); + assert.deepStrictEqual(Object.freeze(result).reexports, ['./final']); + result.reexports[0] = './mutated'; + assert.deepStrictEqual(parse(source).reexports, ['./final']); + assert.throws(() => parse('', '@', { baseline: 'future' }), { name: 'RangeError' }); + assert.throws(() => parse('', '@', { baseline: 1 }), { name: 'TypeError' }); + }); + test('marks filtered UTF-16 names and dependencies incomplete', () => { + assert.deepStrictEqual(parse("module.exports = {}; module.exports = { '\\uD800': 1, valid: 1 };"), + { exports: ['valid'], reexports: [], complete: false }); + assert.deepStrictEqual(parse("module.exports = {}; module.exports = require('./\\uD800');"), + { exports: [], reexports: [], complete: false }); + }); + test('initializes the shared Wasm engine through CommonJS init', async () => { + const rootPath = require.resolve('../lexer.js'), wasmPath = require.resolve('../dist/lexer.js'); + const rootModule = require.cache[rootPath], wasmModule = require.cache[wasmPath]; + const constructor = WebAssembly.Module; + let calls = 0; + delete require.cache[rootPath]; delete require.cache[wasmPath]; + try { + const lexer = require('../lexer.js'); + WebAssembly.Module = function forbiddenSyncCompile () { calls++; throw new Error('Synchronous compilation'); }; + await lexer.init(); + assert.equal(calls, 0); + assert.deepStrictEqual(lexer.parse("module.exports = require('./old'); module.exports = require('./final');") + .reexports, ['./final']); + assert.equal(calls, 0); + } finally { + WebAssembly.Module = constructor; + if (rootModule) require.cache[rootPath] = rootModule; + else delete require.cache[rootPath]; + if (wasmModule) require.cache[wasmPath] = wasmModule; + else delete require.cache[wasmPath]; + } + }); + test('folds pure primitive guards, including escaped strings and typeof', () => { + for (const condition of ['false', '0', "''", 'null', 'void 0', '!true', '1 === 2', "'a' === 'b'", + "'\\x61' !== 'a'", "'\\u{61}' !== 'a'", "'\\141' !== 'a'", "'\\\r\n'", '0x0', '0b0', '0o0']) { + const source = 'if (' + condition + ") module.exports = require('./dead'); else module.exports = require('./live');"; + assert.deepStrictEqual(parse(source), { exports: [], reexports: ['./live'], complete: true }, condition); + } + for (const condition of ['true', '1e2 === 100', '0.1 === 0.1', "typeof 1 === 'number'", + "typeof typeof flag === 'string'", 'require', 'module', 'exports']) { + const source = 'if (' + condition + ") module.exports = require('./live'); else module.exports = require('./dead');"; + assert.deepStrictEqual(parse(source).reexports, ['./live'], condition); + } + assert.deepStrictEqual(parse("module.exports = {}; module.exports = typeof flag ?? require('./dead');").reexports, []); + }); + test('handles nested branches, ternaries, logical guards and logical assignments', () => { + assert.deepStrictEqual(parse("if (first) { if (second) module.exports = require('./a'); " + + "else module.exports = require('./b'); } else module.exports = require('./c');").reexports, + [['./a'], ['./b'], ['./c']]); + assert.deepStrictEqual(parse("module.exports = {}; module.exports = flag ? require('./a') : require('./b');").reexports, + [['./a'], ['./b']]); + for (const operator of ['&&', '||', '??']) { + const result = parse("module.exports = {}; flag " + operator + " (module.exports = require('./active'));"); + assert.equal(result.complete, false); + assert.deepStrictEqual(result.reexports, ['./active']); + } + for (const [operator, initial] of [['&&=', 'true'], ['||=', 'false'], ['??=', 'null']]) { + assert.deepStrictEqual(parse('module.exports = ' + initial + '; module.exports ' + operator + + " require('./active');").reexports, ['./active']); + } + assert.deepStrictEqual(parse("module.exports = require('./a'); module.exports ||= require('./b');").reexports, + [['./a'], ['./b']]); + assert.deepStrictEqual(parse("if (flag) module.exports = require('./a'); else module.exports = require('./b'); " + + "module.exports = require('./final');").reexports, ['./final']); + }); + test('reports only surviving literal names and combined dependency spreads', () => { + assert.deepStrictEqual(parse('module.exports = { removed: 1 }; module.exports = { final: 1 };'), + { exports: ['final'], reexports: [], complete: true }); + assert.deepStrictEqual(parse('module.exports = {}; module.exports = { 0x10: 1, 1e2: 2, 1.5: 3, final: 0, final: 1 };'), + { exports: ['16', '100', '1.5', 'final'], reexports: [], complete: true }); + assert.deepStrictEqual(parse("module.exports = {}; module.exports = { own: require('./value'), " + + "...{ nested: 1, ...require('./spread') }, ...require('./spread') };"), + { exports: ['own', 'nested'], reexports: ['./spread'], complete: true }); + assert.deepStrictEqual(parse("if (flag) module.exports = require('./same'); else module.exports = require(\"./same\");"), + { exports: [], reexports: ['./same'], complete: true }); + assert.deepStrictEqual(parse("module.exports = {}; module.exports = { own: 1 }; " + + "if (module.exports === module.exports) module.exports = require('./live'); else module.exports = require('./dead');") + .reexports, ['./live']); + }); + test('does not guess across unsupported scopes, mutation, or dynamic spreads', () => { + for (const middle of ['for (;;) { break; }', 'while (flag) {}', 'switch (flag) { case 1: break; }', + 'try {} catch (error) {}', 'const flag = true;', 'function unused() { module.exports = {}; }', + 'exports.detached = 1;', 'module.exports.detached = 1;', 'delete module.exports.name;', + 'module.exports = { ...foreign };', 'module.exports = { "__pro\\u0074o__": 1 };', + 'mutate();', 'return;', 'throw new Error();']) { + assert.deepStrictEqual(parse('module.exports = {}; ' + middle + ' module.exports = {};'), + { exports: [], reexports: [], complete: false }, middle); + } + }); + test('does not report a dependency after a CommonJS early return', () => { + assert.deepStrictEqual(parse("module.exports = require('./before'); return; module.exports = require('./dead');"), + { exports: [], reexports: [], complete: false }); + }); + test('invalidates exposed values after opaque reads or dependency execution', () => { + for (const effect of ['flag;', 'flag.name;', "require('./effects');"]) { + assert.deepStrictEqual(parse('module.exports = {}; module.exports = { previous: 1 }; ' + effect), + { exports: [], reexports: [], complete: false }); + } + assert.deepStrictEqual(parse('module.exports = { previous: 1 }; flag; module.exports = { fresh: 1 };'), + { exports: ['fresh'], reexports: [], complete: true }); + }); + test('handles ECMAScript line terminators and semicolon insertion', () => { + for (const line of ['\n', '\r', '\r\n', '\u2028', '\u2029']) { + const before = 'module.exports = {}; module.exports = { before: 1 }'; + const after = 'module.exports = { after: 1 };'; + for (const source of [before + '; // comment' + line + after, before + line + after]) { + assert.deepStrictEqual(parse(source), { exports: ['after'], reexports: [], complete: true }); + } + if (line === '\n' || line === '\r' || line === '\r\n') + assert.deepStrictEqual(parse('#!/usr/bin/env node' + line + before + '; ' + after), + { exports: ['after'], reexports: [], complete: true }); + } + assert.deepStrictEqual(parse('module.exports = {}; module.exports = { after: 1 }; // no line end'), + { exports: ['after'], reexports: [], complete: true }); + }); + test('matches native dependency groups without executing conditions during analysis', () => { + const directory = mkdtempSync(join(tmpdir(), 'cjs-grouped-')); + const filename = join(directory, 'entry.cjs'); + const requireFixture = createRequire(filename); + const previous = Object.getOwnPropertyDescriptor(globalThis, 'groupedFlowFlag'); + let calls = 0, enabled = true; + Object.defineProperty(globalThis, 'groupedFlowFlag', { + configurable: true, get () { calls++; return enabled; } + }); + try { + for (const name of ['a', 'b', 'c', 'd']) writeFileSync(join(directory, name + '.cjs'), 'exports.' + name + ' = 1;'); + const source = "if (groupedFlowFlag) module.exports = {...require('./a.cjs'), ...require('./b.cjs')}; " + + "else module.exports = {...require('./c.cjs'), ...require('./d.cjs')};"; + writeFileSync(filename, source); + const result = parse(source); + assert.equal(calls, 0); + assert.deepStrictEqual(result.reexports, [['./a.cjs', './b.cjs'], ['./c.cjs', './d.cjs']]); + for (const flag of [true, false]) { + enabled = flag; + const observed = requireFixture(filename); + delete requireFixture.cache[requireFixture.resolve(filename)]; + const group = result.reexports[flag ? 0 : 1]; + const keys = new Set(); + for (const dependency of group) for (const key of Object.keys(requireFixture(dependency))) keys.add(key); + assert.deepStrictEqual(Object.keys(observed), [...keys]); + } + assert.equal(calls, 2); + assert.equal(result.complete, true); + } finally { + for (const name of ['a', 'b', 'c', 'd']) { + const dependency = join(directory, name + '.cjs'); + delete requireFixture.cache[requireFixture.resolve(dependency)]; + } + if (previous) Object.defineProperty(globalThis, 'groupedFlowFlag', previous); + else delete globalThis.groupedFlowFlag; + rmSync(directory, { recursive: true, force: true }); + } + }); + test('rejects updates rather than certifying repeated unary operators', () => { + for (const update of ['++module.exports', '--module.exports', 'module.exports++', 'module.exports--']) { + const source = 'module.exports = 0; module.exports = 1; if (' + update + + ' === 2) module.exports = { good: 1 }; else module.exports = { bad: 1 };'; + assert.deepStrictEqual(parse(source), { exports: [], reexports: [], complete: false }, update); + } + assert.deepStrictEqual(parse('module.exports = {}; module.exports = { good: + +1, other: - -1 };'), + { exports: ['good', 'other'], reexports: [], complete: true }); + }); + test('rejects unsupported binary continuations across a line terminator', () => { + for (const separator of ['\n', '\r', '\r\n', '\u2028', '\u2029']) { + for (const operator of ['+', '-']) { + const prefix = "module.exports = {}; module.exports = require('./n')"; + assert.deepStrictEqual(parse(prefix + separator + operator + '1;'), + { exports: [], reexports: [], complete: false }); + assert.deepStrictEqual(parse(prefix + ';' + separator + operator + '1;'), + { exports: [], reexports: ['./n'], complete: true }); + } + } + assert.deepStrictEqual(parse('module.exports = {}; module.exports = 1\n+ 1; ' + + 'if (module.exports === 2) module.exports = { good: 1 }; else module.exports = { bad: 1 };'), + { exports: [], reexports: [], complete: false }); + for (const expression of ['1\nin\n{}', '1\ninstanceof\nObject']) { + assert.deepStrictEqual(parse('module.exports = {}; module.exports = ' + expression + '; ' + + 'if (module.exports) module.exports = { good: 1 }; else module.exports = { bad: 1 };'), + { exports: [], reexports: [], complete: false }); + } + }); + test('bounds tokens, depth, work, and alternative count', () => { + const prefix = 'module.exports = {}; module.exports = {};'; + assert.equal(parse(prefix + ';'.repeat(8192 - 14)).complete, true); + assert.deepStrictEqual(parse(prefix + ';'.repeat(8193 - 14)), + { exports: [], reexports: [], complete: false }); + assert.equal(parse(prefix + '('.repeat(62) + '1' + ')'.repeat(62) + ';').complete, true); + assert.equal(parse(prefix + '('.repeat(63) + '1' + ')'.repeat(63) + ';').complete, false); + assert.equal(parse(prefix + '// ' + 'x'.repeat(65454)).complete, true); + assert.equal(parse(prefix + '// ' + 'x'.repeat(65455)).complete, false); + /** @param {number} start @param {number} end */ + function alternatives (start, end) { + if (end - start === 1) return "require('./" + start + "')"; + const middle = Math.floor((start + end) / 2); + return '(flag ? ' + alternatives(start, middle) + ' : ' + alternatives(middle, end) + ')'; + } + assert.equal(parse('module.exports = {}; module.exports = ' + alternatives(0, 32) + ';').reexports.length, 32); + assert.deepStrictEqual(parse('module.exports = {}; module.exports = ' + alternatives(0, 33) + ';'), + { exports: [], reexports: [], complete: false }); + }); +}); diff --git a/test/integration.js b/test/integration.js index 1a99de9..2eb7f1c 100755 --- a/test/integration.js +++ b/test/integration.js @@ -16,6 +16,9 @@ async function loadParser () { else { parse = require('../lexer.js').parse; } + const parseSource = parse; + parse = /** @param {string} source @param {string} [name] */ (source, name) => + parseSource(source, name, { baseline: 'legacy' }); } const files = fs.readdirSync('test/samples') @@ -29,7 +32,7 @@ const files = fs.readdirSync('test/samples') }; }); -suite('Samples', () => { +suite('Legacy samples', () => { suiteSetup(async () => await loadParser()); const selfSource = fs.readFileSync(process.cwd() + '/lexer.js').toString();