From bed7ed1e8b528d0b98280439362526e37fbebcb4 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Mon, 31 Aug 2026 11:34:04 -0400 Subject: [PATCH 1/2] fix(dataflow): resolve each callable under its owning tsconfig (INCOMPLETE) Extraction indexed every callable against the ROOT program alone, so a file a deeper program owns was absent from that index and hit `if (!fn) continue` -- skipped silently, not mis-resolved. On vscode (92 programs, no root tsconfig) that extracted 1,204 of 174,767 callables. This threads BuiltProgram[] through startExtraction so a callable is looked up in the program that owns its file, matching what the symbol table and the L2 call graph already do. Worker tasks are partitioned within a program so each task's files share one tsconfig -- previously every task got a single config, which is also why -j N and -j 1 could diverge on multi-program repos. Adds a coverage line so a near-empty L3 can no longer pass for a successful one. INCOMPLETE -- DO NOT MERGE. Correct on the multi-tsconfig fixture (break-checked: reverting to root-only lookup fails the new test) but it does not fit in memory at vscode scale. Walking a project forces tsc to parse and bind every file in it; previously only the root program was ever walked, so the other 91 stayed lazy. Three runs, all killed by a JSC heap OOM (exit 133): sequential 26.9GB, sequential with one index resident at a time 28.6GB, -j 4 29.2GB. Bounding the index did not help because the memory is in the Projects, not the index. Freeing them needs core.ts to stop running extraction concurrently with the call-graph solve (both hold every program live), or a project pool that materializes a bounded number at a time. See #111. --- src/core.ts | 14 ++-- src/dataflow/index.ts | 136 ++++++++++++++++++++++++++++-------- test/multi-tsconfig.test.ts | 34 +++++++++ 3 files changed, 144 insertions(+), 40 deletions(-) diff --git a/src/core.ts b/src/core.ts index 48aec15..6820b37 100644 --- a/src/core.ts +++ b/src/core.ts @@ -35,15 +35,11 @@ export async function analyze(opts: AnalysisOptions): Promise { // extraction doesn't need callee resolution, so the two run concurrently (the contract's // "points-to solve runs concurrently with stages 1–4") and join in buildProgramGraphs. // - // Multi-program (#56) NOTE: each BuiltProgram carries its owning tsconfig (`configPath`), so the - // file→program config map exists here. Threading it per-file into the dataflow workers (each - // builds its own Project from ONE tsconfig, src/dataflow/worker.ts) is deferred: extraction still - // uses the root program's config, so at L3 files in a NESTED program are analyzed with the root - // options. Documented limitation — L2 call-graph resolution (the #56 gate) is fully per-program. - if (opts.analysisLevel >= 3 && programs.length > 1) { - log.warn(`L3 dataflow uses the root tsconfig for all ${programs.length} programs; nested-program files may under-resolve (see #56)`); - } - const extraction = opts.analysisLevel >= 3 ? startExtraction(project, symbol_table, mat.tsConfigFilePath, opts, log) : null; + // Multi-program: extraction resolves each callable under the tsconfig that OWNS its file, the + // same assignment the symbol table and the L2 call graph use (#111). It used to index the whole + // repo against the root program alone, which silently skipped every callable a deeper program + // owned. + const extraction = opts.analysisLevel >= 3 ? startExtraction(programs, symbol_table, opts, log) : null; // Call graph: the tsc resolver, per program (each with its own Project + its slice of callables // via `only`), merged across programs. Only worth running at level >= 2: finalizeAnalysis diff --git a/src/dataflow/index.ts b/src/dataflow/index.ts index 51ded96..fc15a97 100644 --- a/src/dataflow/index.ts +++ b/src/dataflow/index.ts @@ -26,7 +26,8 @@ */ import * as fs from "node:fs"; import * as path from "node:path"; -import type { Project } from "ts-morph"; +import type { Node } from "ts-morph"; +import type { BuiltProgram } from "../syntactic_analysis/symbolTable"; import type { AnalysisOptions } from "../options"; import { PROGRAM_GRAPHS_SCHEMA_VERSION, @@ -66,13 +67,18 @@ export interface ExtractionHandle { } export function startExtraction( - project: Project, + programs: BuiltProgram[], symbol_table: Record, - tsConfigFilePath: string | null, opts: AnalysisOptions, log: Logger, ): ExtractionHandle { const callables = collectCallables(symbol_table); + // Every callable is extracted under the tsconfig that OWNS its file, exactly as the symbol table + // and the L2 call graph already resolve it (#111). Indexing the whole repo against one root + // program silently drops every callable a deeper program owns: `indexCallableDecls` never sees + // the declaration, and the `if (!fn) continue` below skips it. On vscode -- 92 programs, no root + // tsconfig -- that was 1,204 of 174,767 callables extracted. + const ownerOf = programOwnerIndex(programs, callables, opts.input); // Partition callables by owning file (round-robin over the sorted file list) so each worker // deeply visits only its share of the program. TSCallable.abs_path is the declaration's ABSOLUTE @@ -95,7 +101,7 @@ export function startExtraction( const jobs = opts.jobs === 0 ? 1 : opts.jobs; if (jobs <= 1) { - return { promise: Promise.resolve(extractSequential(project, callables, opts)), pool: null }; + return { promise: Promise.resolve(extractSequential(programs, ownerOf, callables, opts, log)), pool: null }; } const workerCount = Math.max(1, Math.min(jobs, files.length)); @@ -104,30 +110,44 @@ export function startExtraction( pool = new WorkerPool(workerCount); } catch (e) { log.warn(`graph workers unavailable (${(e as Error).message}); extracting sequentially`); - return { promise: Promise.resolve(extractSequential(project, callables, opts)), pool: null }; + return { promise: Promise.resolve(extractSequential(programs, ownerOf, callables, opts, log)), pool: null }; } - const partitions: Array> = Array.from( - { length: workerCount }, - () => [], - ); - files.forEach((f, i) => partitions[i % workerCount]?.push(...(byFile.get(f) ?? []))); + // Partition WITHIN each owning program: a task's files must all share one tsconfig, because the + // worker builds a single Project per task config (#111). Round-robin inside a program keeps the + // per-worker balance the previous global round-robin had. + const byConfig = new Map(); + for (const f of files) { + const configPath = ownerOf.get(f) ?? null; + const key = configPath ?? ""; + const g = byConfig.get(key) ?? { configPath, files: [] }; + g.files.push(f); + byConfig.set(key, g); + } + const partitions: Array<{ configPath: string | null; sigs: Array<{ signature: string; path: string; absPath: string }> }> = []; + for (const key of [...byConfig.keys()].sort()) { + const g = byConfig.get(key)!; + const slots: Array> = Array.from( + { length: Math.max(1, Math.min(workerCount, g.files.length)) }, + () => [], + ); + g.files.forEach((f, i) => slots[i % slots.length]?.push(...(byFile.get(f) ?? []))); + for (const s of slots) if (s.length) partitions.push({ configPath: g.configPath, sigs: s }); + } const handle: ExtractionHandle = { promise: Promise.resolve(new Map()), pool }; handle.promise = Promise.all( - partitions - .filter((p) => p.length) - .map((sigs) => { - const task: ExtractTask = { - type: "extract", - root: opts.input, - tsConfigFilePath, - skipTests: opts.skipTests, - k: opts.graphFieldDepth, - sigs, - }; - return pool.exec(task); - }), + partitions.map(({ configPath, sigs }) => { + const task: ExtractTask = { + type: "extract", + root: opts.input, + tsConfigFilePath: configPath, + skipTests: opts.skipTests, + k: opts.graphFieldDepth, + sigs, + }; + return pool.exec(task); + }), ) .then((chunks) => { const out = new Map(); @@ -137,6 +157,7 @@ export function startExtraction( // main thread's — treat it as a failure, never as an empty program. throw new Error("workers returned no callables"); } + reportCoverage(out.size, callables.size, log); return out; }) .catch((e: Error) => { @@ -146,28 +167,81 @@ export function startExtraction( log.warn(`graph extraction workers failed (${e.message}); falling back to sequential`); handle.pool?.close(); handle.pool = null; - return extractSequential(project, callables, opts); + return extractSequential(programs, ownerOf, callables, opts, log); }); return handle; } +/** + * absolute file path -> the tsconfig that OWNS it, taken from the same program assignment the + * symbol table used (`BuiltProgram.fileKeys`, deepest scope wins). A file no program claims maps + * to null, the default-options program. + */ +function programOwnerIndex( + programs: BuiltProgram[], + callables: Map, + root: string, +): Map { + const byFileKey = new Map(); + for (const p of programs) for (const k of p.fileKeys) if (!byFileKey.has(k)) byFileKey.set(k, p.configPath); + const out = new Map(); + for (const c of callables.values()) { + if (out.has(c.abs_path)) continue; + out.set(c.abs_path, byFileKey.get(fileKeyOf(c.abs_path, root).fileKey) ?? null); + } + return out; +} + function extractSequential( - project: Project, + programs: BuiltProgram[], + ownerOf: Map, callables: Map, opts: AnalysisOptions, + log: Logger, ): Map { - const astIndex = indexCallableDecls(project, opts.input); - const out = new Map(); + // Group first, then index ONE program at a time. Building all of them up front holds every + // program's declaration nodes live at once: on vscode (92 programs) that peaked at 26.9GB and + // the process was killed. Only one index is resident here, so the memory profile matches the + // single-root version while the lookup is per-owning-program (#111). + const byConfig = new Map>(); for (const [sig, c] of [...callables.entries()].sort(([a], [b]) => a.localeCompare(b))) { - const fn = astIndex.get(sig); - if (!fn) continue; // bodiless (interface/abstract/ambient/implicit) or unmatchable - const data = extractCallableData(sig, fn, fileKeyOf(c.abs_path, opts.input).fileKey, opts.input, opts.graphFieldDepth); - if (data) out.set(sig, data); + const key = ownerOf.get(c.abs_path) ?? ""; + const group = byConfig.get(key); + if (group) group.push([sig, c]); + else byConfig.set(key, [[sig, c]]); } + + const out = new Map(); + for (const p of programs) { + const group = byConfig.get(p.configPath ?? ""); + if (!group?.length) continue; + const idx = indexCallableDecls(p.project, opts.input); + for (const [sig, c] of group) { + const fn = idx.get(sig); + if (!fn) continue; // bodiless (interface/abstract/ambient/implicit) or unmatchable + const data = extractCallableData(sig, fn, fileKeyOf(c.abs_path, opts.input).fileKey, opts.input, opts.graphFieldDepth); + if (data) out.set(sig, data); + } + // idx drops here — the next program's index replaces it rather than accumulating. + } + reportCoverage(out.size, callables.size, log); return out; } +/** + * Say how much flow was actually extracted. A near-empty L3 used to be indistinguishable from a + * successful one -- #111 shipped precisely because nothing reported that 0.7% of callables had + * been populated. + */ +function reportCoverage(extracted: number, collected: number, log: Logger): void { + if (!collected) return; + const pct = (100 * extracted) / collected; + const msg = `dataflow: extracted ${extracted.toLocaleString()} of ${collected.toLocaleString()} callables (${pct.toFixed(1)}%)`; + if (pct < 50) log.warn(`${msg} — most callables produced no control/data flow`); + else log.info(msg); +} + // ------------------------------------------------------------------------------------------------ // Stages 5–7 + emission // ------------------------------------------------------------------------------------------------ diff --git a/test/multi-tsconfig.test.ts b/test/multi-tsconfig.test.ts index 2d630f9..49bb757 100644 --- a/test/multi-tsconfig.test.ts +++ b/test/multi-tsconfig.test.ts @@ -56,6 +56,16 @@ async function run(): Promise { } } +async function runLevel3(): Promise { + const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), "cants-multitsconfig-l3-")); + try { + const r = await analyze({ ...options(), analysisLevel: 3, graphs: ["cfg", "dfg", "pdg"], cacheDir }); + return r.application.application; + } finally { + fs.rmSync(cacheDir, { recursive: true, force: true }); + } +} + describe("multi-tsconfig program construction (#56)", () => { test("both programs' files are in one merged symbol table, each owned once", async () => { const app = await run(); @@ -89,6 +99,30 @@ describe("multi-tsconfig program construction (#56)", () => { expect(Object.keys(app.external_symbols ?? {})).not.toContain("@app/service.svc"); }); + // #111: extraction used to index every callable against the ROOT program alone. A file a DEEPER + // program owns is absent from that index, so `if (!fn) continue` skipped it silently — the run + // succeeded and produced no flow. Measured on vscode (92 programs, no root tsconfig): 1,204 of + // 174,767 callables. Here the root tsconfig excludes `web`, so every web callable is owned by + // the nested program and is exactly the case that vanished. + test("L3 populates callables owned by the NESTED program, not just the root's (#111)", async () => { + const app = await runLevel3(); + + const web = app.symbol_table["web/src/main.ts"]; + expect(web, "web/src/main.ts missing from the symbol table").toBeDefined(); + const boot = (web!.functions ?? {})["boot"]; + expect(boot, "boot() missing").toBeDefined(); + expect(boot!.cfg?.length ?? 0, "nested-program callable got no CFG").toBeGreaterThan(0); + expect(Object.keys(boot!.body ?? {}).some((k) => k === "@entry")).toBe(true); + + const svcMod = app.symbol_table["web/src/app/service.ts"]; + const svc = (svcMod?.functions ?? {})["svc"]; + expect(svc?.cfg?.length ?? 0, "nested-program callee got no CFG").toBeGreaterThan(0); + + // The root program's own callables keep working — this is a widening, not a swap. + const server = (app.symbol_table["src/server.ts"]?.functions ?? {})["serve"]; + expect(server?.cfg?.length ?? 0, "root-program callable lost its CFG").toBeGreaterThan(0); + }); + test("the root program's relative import still resolves (no regression)", async () => { const app = await run(); const edge = app.call_graph.find((e) => e.source === "src/server.serve"); From ab3cfd0c776d95ea7d298af529aa015a6a2668ef Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 2 Sep 2026 10:25:19 -0400 Subject: [PATCH 2/2] perf(dataflow): dispose programs after extraction; stream the JSON emit Two memory ceilings on top of the per-program extraction fix, both from #112. Step 2 -- bound the resident program set. Extraction now runs AFTER the call-graph solve rather than "concurrently" with it. That costs nothing: at the default -j 1, startExtraction evaluated extractSequential eagerly, so the two were already serial and the concurrency comment described only -j N > 1. Ordering them explicitly makes extraction the last reader of every program's Project, which is what lets each one's source files be released as extraction finishes with it. The root project is spared -- finalizeAnalysis still needs it for the config-use dataflow tier. Measured on vscode: peak 29.2GB -> 24.0GB. Step 1 -- stream the JSON emit. `JSON.stringify(application)` materialised the entire output before writing a byte, so peak carried the tree and its serialisation at once and a large enough analysis exceeded the runtime's maximum string length outright (vscode -a 4: `RangeError: Out of memory` in stringify). writeAnalysisJson walks the envelope in insertion order and streams `symbol_table` per module and the application-scope arrays per element, holding at most one of either as a string. Byte-identical output, verified against the previous whole-string path on six fixtures at -a 4 with every graph selector -- the first cut hoisted symbol_table to the front and produced the same bytes in a different key order, which the comparison caught. vscode at -a 4 STILL does not complete: 30.8GB, killed. The earlier run reporting a stringify error at 24.0GB was not "the analysis finished, only emit failed" -- JS threw a catchable RangeError while already near the ceiling, and removing that early exit only let the run continue until the OS killed it. Both changes are real improvements and neither is sufficient there. What they do buy, measured on superset-frontend (39 programs, 1,841 modules): callables with CFG 8,138 (73.4%) -> 11,023 (99.4%), param_in 6,475 -> 17,920, for +0.3GB and no extra wall-clock. --- src/core.ts | 9 +++++- src/dataflow/index.ts | 18 ++++++++---- src/utils/serialize.ts | 63 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/src/core.ts b/src/core.ts index 6820b37..cc12435 100644 --- a/src/core.ts +++ b/src/core.ts @@ -39,7 +39,6 @@ export async function analyze(opts: AnalysisOptions): Promise { // same assignment the symbol table and the L2 call graph use (#111). It used to index the whole // repo against the root program alone, which silently skipped every callable a deeper program // owned. - const extraction = opts.analysisLevel >= 3 ? startExtraction(programs, symbol_table, opts, log) : null; // Call graph: the tsc resolver, per program (each with its own Project + its slice of callables // via `only`), merged across programs. Only worth running at level >= 2: finalizeAnalysis @@ -73,6 +72,14 @@ export async function analyze(opts: AnalysisOptions): Promise { } } } + // Extraction runs AFTER the solve, not concurrently with it (#112). The two only ever + // overlapped under `-j N > 1` -- at the default `-j 1` startExtraction evaluates + // extractSequential eagerly, so they were already serial. Ordering them explicitly lets + // extraction be the LAST reader of each program's Project, which is what makes disposal safe: + // on a repo with many programs, holding all of them materialised is what exhausted the heap. + const extraction = + opts.analysisLevel >= 3 ? startExtraction(programs, symbol_table, opts, log, project) : null; + const call_graph = cg.edges; // Repository-artifact layer (#101, python PR #160 parity): level-free, identical at every -a. diff --git a/src/dataflow/index.ts b/src/dataflow/index.ts index fc15a97..ac599df 100644 --- a/src/dataflow/index.ts +++ b/src/dataflow/index.ts @@ -26,7 +26,7 @@ */ import * as fs from "node:fs"; import * as path from "node:path"; -import type { Node } from "ts-morph"; +import type { Node, Project } from "ts-morph"; import type { BuiltProgram } from "../syntactic_analysis/symbolTable"; import type { AnalysisOptions } from "../options"; import { @@ -71,6 +71,7 @@ export function startExtraction( symbol_table: Record, opts: AnalysisOptions, log: Logger, + keepProject?: Project, ): ExtractionHandle { const callables = collectCallables(symbol_table); // Every callable is extracted under the tsconfig that OWNS its file, exactly as the symbol table @@ -101,7 +102,7 @@ export function startExtraction( const jobs = opts.jobs === 0 ? 1 : opts.jobs; if (jobs <= 1) { - return { promise: Promise.resolve(extractSequential(programs, ownerOf, callables, opts, log)), pool: null }; + return { promise: Promise.resolve(extractSequential(programs, ownerOf, callables, opts, log, keepProject)), pool: null }; } const workerCount = Math.max(1, Math.min(jobs, files.length)); @@ -110,7 +111,7 @@ export function startExtraction( pool = new WorkerPool(workerCount); } catch (e) { log.warn(`graph workers unavailable (${(e as Error).message}); extracting sequentially`); - return { promise: Promise.resolve(extractSequential(programs, ownerOf, callables, opts, log)), pool: null }; + return { promise: Promise.resolve(extractSequential(programs, ownerOf, callables, opts, log, keepProject)), pool: null }; } // Partition WITHIN each owning program: a task's files must all share one tsconfig, because the @@ -167,7 +168,7 @@ export function startExtraction( log.warn(`graph extraction workers failed (${e.message}); falling back to sequential`); handle.pool?.close(); handle.pool = null; - return extractSequential(programs, ownerOf, callables, opts, log); + return extractSequential(programs, ownerOf, callables, opts, log, keepProject); }); return handle; @@ -199,6 +200,7 @@ function extractSequential( callables: Map, opts: AnalysisOptions, log: Logger, + keepProject?: Project, ): Map { // Group first, then index ONE program at a time. Building all of them up front holds every // program's declaration nodes live at once: on vscode (92 programs) that peaked at 26.9GB and @@ -223,7 +225,13 @@ function extractSequential( const data = extractCallableData(sig, fn, fileKeyOf(c.abs_path, opts.input).fileKey, opts.input, opts.graphFieldDepth); if (data) out.set(sig, data); } - // idx drops here — the next program's index replaces it rather than accumulating. + // Release this program's ASTs now that nothing else will read them (#112). Indexing a + // project forces tsc to parse and bind every file in it, so on a repo with many programs the + // materialised set is the dominant cost -- vscode has 92. `keepProject` is the root program, + // which finalizeAnalysis still needs for the config-use dataflow tier, so it is spared. + if (p.project !== keepProject) { + for (const sf of p.project.getSourceFiles()) p.project.removeSourceFile(sf); + } } reportCoverage(out.size, callables.size, log); return out; diff --git a/src/utils/serialize.ts b/src/utils/serialize.ts index 95b9989..4b5198d 100644 --- a/src/utils/serialize.ts +++ b/src/utils/serialize.ts @@ -23,7 +23,68 @@ export async function emit(application: TSAnalysis, opts: AnalysisOptions): Prom return; } fs.mkdirSync(opts.output, { recursive: true }); - fs.writeFileSync(path.join(opts.output, "analysis.json"), JSON.stringify(application)); + writeAnalysisJson(path.join(opts.output, "analysis.json"), application); +} + +/** + * Write the envelope WITHOUT ever holding it as one string (#112). + * + * `JSON.stringify(application)` materialises the entire output before a byte is written, so peak + * memory carries the tree AND its serialisation at once, and a large enough analysis exceeds the + * runtime's maximum string length outright: vscode at -a 4 dies with `RangeError: Out of memory` + * in stringify after the analysis itself has completed successfully. + * + * The envelope's shape is fixed and its two unbounded members are `symbol_table` (keyed by module) + * and the application-scope edge arrays, so both are streamed element by element. Each element is + * still stringified individually — bounded by the largest single module, not by the whole repo. + * Output is byte-identical to the previous whole-string write. + */ +function writeAnalysisJson(file: string, envelope: TSAnalysis): void { + const fd = fs.openSync(file, "w"); + const put = (chunk: string): void => { + fs.writeSync(fd, chunk); + }; + try { + const { application: app, ...head } = envelope as unknown as Record; + const root = app as Record; + + // envelope head — every key except `application` + put("{"); + for (const [k, v] of Object.entries(head)) put(`${JSON.stringify(k)}:${JSON.stringify(v)},`); + put(`"application":{`); + + // Walk the application's keys in INSERTION ORDER so the bytes match what JSON.stringify + // produced; only the two unbounded members stream, everything else is small. + let firstKey = true; + for (const [k, v] of Object.entries(root)) { + if (v === undefined) continue; + if (!firstKey) put(","); + firstKey = false; + put(`${JSON.stringify(k)}:`); + if (k === "symbol_table" && v && typeof v === "object") { + put("{"); + let first = true; + for (const [key, mod] of Object.entries(v as Record)) { + if (!first) put(","); + first = false; + put(`${JSON.stringify(key)}:${JSON.stringify(mod)}`); + } + put("}"); + } else if (Array.isArray(v)) { + put("["); + for (let i = 0; i < v.length; i++) { + if (i) put(","); + put(JSON.stringify(v[i])); + } + put("]"); + } else { + put(JSON.stringify(v)); + } + } + put("}}"); + } finally { + fs.closeSync(fd); + } } /**