diff --git a/src/core.ts b/src/core.ts index 48aec15..cc12435 100644 --- a/src/core.ts +++ b/src/core.ts @@ -35,15 +35,10 @@ export async function analyze(opts: AnalysisOptions): Promise { // extraction doesn't need callee resolution, so the two run concurrently (the contract's // "points-to solve runs concurrently with stages 1–4") and join in buildProgramGraphs. // - // Multi-program (#56) NOTE: each BuiltProgram carries its owning tsconfig (`configPath`), so the - // file→program config map exists here. Threading it per-file into the dataflow workers (each - // builds its own Project from ONE tsconfig, src/dataflow/worker.ts) is deferred: extraction still - // uses the root program's config, so at L3 files in a NESTED program are analyzed with the root - // options. Documented limitation — L2 call-graph resolution (the #56 gate) is fully per-program. - if (opts.analysisLevel >= 3 && programs.length > 1) { - log.warn(`L3 dataflow uses the root tsconfig for all ${programs.length} programs; nested-program files may under-resolve (see #56)`); - } - const extraction = opts.analysisLevel >= 3 ? startExtraction(project, symbol_table, mat.tsConfigFilePath, opts, log) : null; + // Multi-program: extraction resolves each callable under the tsconfig that OWNS its file, the + // same assignment the symbol table and the L2 call graph use (#111). It used to index the whole + // repo against the root program alone, which silently skipped every callable a deeper program + // owned. // Call graph: the tsc resolver, per program (each with its own Project + its slice of callables // via `only`), merged across programs. Only worth running at level >= 2: finalizeAnalysis @@ -77,6 +72,14 @@ export async function analyze(opts: AnalysisOptions): Promise { } } } + // Extraction runs AFTER the solve, not concurrently with it (#112). The two only ever + // overlapped under `-j N > 1` -- at the default `-j 1` startExtraction evaluates + // extractSequential eagerly, so they were already serial. Ordering them explicitly lets + // extraction be the LAST reader of each program's Project, which is what makes disposal safe: + // on a repo with many programs, holding all of them materialised is what exhausted the heap. + const extraction = + opts.analysisLevel >= 3 ? startExtraction(programs, symbol_table, opts, log, project) : null; + const call_graph = cg.edges; // Repository-artifact layer (#101, python PR #160 parity): level-free, identical at every -a. diff --git a/src/dataflow/index.ts b/src/dataflow/index.ts index 51ded96..ac599df 100644 --- a/src/dataflow/index.ts +++ b/src/dataflow/index.ts @@ -26,7 +26,8 @@ */ import * as fs from "node:fs"; import * as path from "node:path"; -import type { Project } from "ts-morph"; +import type { Node, Project } from "ts-morph"; +import type { BuiltProgram } from "../syntactic_analysis/symbolTable"; import type { AnalysisOptions } from "../options"; import { PROGRAM_GRAPHS_SCHEMA_VERSION, @@ -66,13 +67,19 @@ export interface ExtractionHandle { } export function startExtraction( - project: Project, + programs: BuiltProgram[], symbol_table: Record, - tsConfigFilePath: string | null, opts: AnalysisOptions, log: Logger, + keepProject?: Project, ): ExtractionHandle { const callables = collectCallables(symbol_table); + // Every callable is extracted under the tsconfig that OWNS its file, exactly as the symbol table + // and the L2 call graph already resolve it (#111). Indexing the whole repo against one root + // program silently drops every callable a deeper program owns: `indexCallableDecls` never sees + // the declaration, and the `if (!fn) continue` below skips it. On vscode -- 92 programs, no root + // tsconfig -- that was 1,204 of 174,767 callables extracted. + const ownerOf = programOwnerIndex(programs, callables, opts.input); // Partition callables by owning file (round-robin over the sorted file list) so each worker // deeply visits only its share of the program. TSCallable.abs_path is the declaration's ABSOLUTE @@ -95,7 +102,7 @@ export function startExtraction( const jobs = opts.jobs === 0 ? 1 : opts.jobs; if (jobs <= 1) { - return { promise: Promise.resolve(extractSequential(project, callables, opts)), pool: null }; + return { promise: Promise.resolve(extractSequential(programs, ownerOf, callables, opts, log, keepProject)), pool: null }; } const workerCount = Math.max(1, Math.min(jobs, files.length)); @@ -104,30 +111,44 @@ export function startExtraction( pool = new WorkerPool(workerCount); } catch (e) { log.warn(`graph workers unavailable (${(e as Error).message}); extracting sequentially`); - return { promise: Promise.resolve(extractSequential(project, callables, opts)), pool: null }; + return { promise: Promise.resolve(extractSequential(programs, ownerOf, callables, opts, log, keepProject)), pool: null }; } - const partitions: Array> = Array.from( - { length: workerCount }, - () => [], - ); - files.forEach((f, i) => partitions[i % workerCount]?.push(...(byFile.get(f) ?? []))); + // Partition WITHIN each owning program: a task's files must all share one tsconfig, because the + // worker builds a single Project per task config (#111). Round-robin inside a program keeps the + // per-worker balance the previous global round-robin had. + const byConfig = new Map(); + for (const f of files) { + const configPath = ownerOf.get(f) ?? null; + const key = configPath ?? ""; + const g = byConfig.get(key) ?? { configPath, files: [] }; + g.files.push(f); + byConfig.set(key, g); + } + const partitions: Array<{ configPath: string | null; sigs: Array<{ signature: string; path: string; absPath: string }> }> = []; + for (const key of [...byConfig.keys()].sort()) { + const g = byConfig.get(key)!; + const slots: Array> = Array.from( + { length: Math.max(1, Math.min(workerCount, g.files.length)) }, + () => [], + ); + g.files.forEach((f, i) => slots[i % slots.length]?.push(...(byFile.get(f) ?? []))); + for (const s of slots) if (s.length) partitions.push({ configPath: g.configPath, sigs: s }); + } const handle: ExtractionHandle = { promise: Promise.resolve(new Map()), pool }; handle.promise = Promise.all( - partitions - .filter((p) => p.length) - .map((sigs) => { - const task: ExtractTask = { - type: "extract", - root: opts.input, - tsConfigFilePath, - skipTests: opts.skipTests, - k: opts.graphFieldDepth, - sigs, - }; - return pool.exec(task); - }), + partitions.map(({ configPath, sigs }) => { + const task: ExtractTask = { + type: "extract", + root: opts.input, + tsConfigFilePath: configPath, + skipTests: opts.skipTests, + k: opts.graphFieldDepth, + sigs, + }; + return pool.exec(task); + }), ) .then((chunks) => { const out = new Map(); @@ -137,6 +158,7 @@ export function startExtraction( // main thread's — treat it as a failure, never as an empty program. throw new Error("workers returned no callables"); } + reportCoverage(out.size, callables.size, log); return out; }) .catch((e: Error) => { @@ -146,28 +168,88 @@ export function startExtraction( log.warn(`graph extraction workers failed (${e.message}); falling back to sequential`); handle.pool?.close(); handle.pool = null; - return extractSequential(project, callables, opts); + return extractSequential(programs, ownerOf, callables, opts, log, keepProject); }); return handle; } +/** + * absolute file path -> the tsconfig that OWNS it, taken from the same program assignment the + * symbol table used (`BuiltProgram.fileKeys`, deepest scope wins). A file no program claims maps + * to null, the default-options program. + */ +function programOwnerIndex( + programs: BuiltProgram[], + callables: Map, + root: string, +): Map { + const byFileKey = new Map(); + for (const p of programs) for (const k of p.fileKeys) if (!byFileKey.has(k)) byFileKey.set(k, p.configPath); + const out = new Map(); + for (const c of callables.values()) { + if (out.has(c.abs_path)) continue; + out.set(c.abs_path, byFileKey.get(fileKeyOf(c.abs_path, root).fileKey) ?? null); + } + return out; +} + function extractSequential( - project: Project, + programs: BuiltProgram[], + ownerOf: Map, callables: Map, opts: AnalysisOptions, + log: Logger, + keepProject?: Project, ): Map { - const astIndex = indexCallableDecls(project, opts.input); - const out = new Map(); + // Group first, then index ONE program at a time. Building all of them up front holds every + // program's declaration nodes live at once: on vscode (92 programs) that peaked at 26.9GB and + // the process was killed. Only one index is resident here, so the memory profile matches the + // single-root version while the lookup is per-owning-program (#111). + const byConfig = new Map>(); for (const [sig, c] of [...callables.entries()].sort(([a], [b]) => a.localeCompare(b))) { - const fn = astIndex.get(sig); - if (!fn) continue; // bodiless (interface/abstract/ambient/implicit) or unmatchable - const data = extractCallableData(sig, fn, fileKeyOf(c.abs_path, opts.input).fileKey, opts.input, opts.graphFieldDepth); - if (data) out.set(sig, data); + const key = ownerOf.get(c.abs_path) ?? ""; + const group = byConfig.get(key); + if (group) group.push([sig, c]); + else byConfig.set(key, [[sig, c]]); } + + const out = new Map(); + for (const p of programs) { + const group = byConfig.get(p.configPath ?? ""); + if (!group?.length) continue; + const idx = indexCallableDecls(p.project, opts.input); + for (const [sig, c] of group) { + const fn = idx.get(sig); + if (!fn) continue; // bodiless (interface/abstract/ambient/implicit) or unmatchable + const data = extractCallableData(sig, fn, fileKeyOf(c.abs_path, opts.input).fileKey, opts.input, opts.graphFieldDepth); + if (data) out.set(sig, data); + } + // Release this program's ASTs now that nothing else will read them (#112). Indexing a + // project forces tsc to parse and bind every file in it, so on a repo with many programs the + // materialised set is the dominant cost -- vscode has 92. `keepProject` is the root program, + // which finalizeAnalysis still needs for the config-use dataflow tier, so it is spared. + if (p.project !== keepProject) { + for (const sf of p.project.getSourceFiles()) p.project.removeSourceFile(sf); + } + } + reportCoverage(out.size, callables.size, log); return out; } +/** + * Say how much flow was actually extracted. A near-empty L3 used to be indistinguishable from a + * successful one -- #111 shipped precisely because nothing reported that 0.7% of callables had + * been populated. + */ +function reportCoverage(extracted: number, collected: number, log: Logger): void { + if (!collected) return; + const pct = (100 * extracted) / collected; + const msg = `dataflow: extracted ${extracted.toLocaleString()} of ${collected.toLocaleString()} callables (${pct.toFixed(1)}%)`; + if (pct < 50) log.warn(`${msg} — most callables produced no control/data flow`); + else log.info(msg); +} + // ------------------------------------------------------------------------------------------------ // Stages 5–7 + emission // ------------------------------------------------------------------------------------------------ diff --git a/src/utils/serialize.ts b/src/utils/serialize.ts index 95b9989..4b5198d 100644 --- a/src/utils/serialize.ts +++ b/src/utils/serialize.ts @@ -23,7 +23,68 @@ export async function emit(application: TSAnalysis, opts: AnalysisOptions): Prom return; } fs.mkdirSync(opts.output, { recursive: true }); - fs.writeFileSync(path.join(opts.output, "analysis.json"), JSON.stringify(application)); + writeAnalysisJson(path.join(opts.output, "analysis.json"), application); +} + +/** + * Write the envelope WITHOUT ever holding it as one string (#112). + * + * `JSON.stringify(application)` materialises the entire output before a byte is written, so peak + * memory carries the tree AND its serialisation at once, and a large enough analysis exceeds the + * runtime's maximum string length outright: vscode at -a 4 dies with `RangeError: Out of memory` + * in stringify after the analysis itself has completed successfully. + * + * The envelope's shape is fixed and its two unbounded members are `symbol_table` (keyed by module) + * and the application-scope edge arrays, so both are streamed element by element. Each element is + * still stringified individually — bounded by the largest single module, not by the whole repo. + * Output is byte-identical to the previous whole-string write. + */ +function writeAnalysisJson(file: string, envelope: TSAnalysis): void { + const fd = fs.openSync(file, "w"); + const put = (chunk: string): void => { + fs.writeSync(fd, chunk); + }; + try { + const { application: app, ...head } = envelope as unknown as Record; + const root = app as Record; + + // envelope head — every key except `application` + put("{"); + for (const [k, v] of Object.entries(head)) put(`${JSON.stringify(k)}:${JSON.stringify(v)},`); + put(`"application":{`); + + // Walk the application's keys in INSERTION ORDER so the bytes match what JSON.stringify + // produced; only the two unbounded members stream, everything else is small. + let firstKey = true; + for (const [k, v] of Object.entries(root)) { + if (v === undefined) continue; + if (!firstKey) put(","); + firstKey = false; + put(`${JSON.stringify(k)}:`); + if (k === "symbol_table" && v && typeof v === "object") { + put("{"); + let first = true; + for (const [key, mod] of Object.entries(v as Record)) { + if (!first) put(","); + first = false; + put(`${JSON.stringify(key)}:${JSON.stringify(mod)}`); + } + put("}"); + } else if (Array.isArray(v)) { + put("["); + for (let i = 0; i < v.length; i++) { + if (i) put(","); + put(JSON.stringify(v[i])); + } + put("]"); + } else { + put(JSON.stringify(v)); + } + } + put("}}"); + } finally { + fs.closeSync(fd); + } } /** diff --git a/test/multi-tsconfig.test.ts b/test/multi-tsconfig.test.ts index 2d630f9..49bb757 100644 --- a/test/multi-tsconfig.test.ts +++ b/test/multi-tsconfig.test.ts @@ -56,6 +56,16 @@ async function run(): Promise { } } +async function runLevel3(): Promise { + const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), "cants-multitsconfig-l3-")); + try { + const r = await analyze({ ...options(), analysisLevel: 3, graphs: ["cfg", "dfg", "pdg"], cacheDir }); + return r.application.application; + } finally { + fs.rmSync(cacheDir, { recursive: true, force: true }); + } +} + describe("multi-tsconfig program construction (#56)", () => { test("both programs' files are in one merged symbol table, each owned once", async () => { const app = await run(); @@ -89,6 +99,30 @@ describe("multi-tsconfig program construction (#56)", () => { expect(Object.keys(app.external_symbols ?? {})).not.toContain("@app/service.svc"); }); + // #111: extraction used to index every callable against the ROOT program alone. A file a DEEPER + // program owns is absent from that index, so `if (!fn) continue` skipped it silently — the run + // succeeded and produced no flow. Measured on vscode (92 programs, no root tsconfig): 1,204 of + // 174,767 callables. Here the root tsconfig excludes `web`, so every web callable is owned by + // the nested program and is exactly the case that vanished. + test("L3 populates callables owned by the NESTED program, not just the root's (#111)", async () => { + const app = await runLevel3(); + + const web = app.symbol_table["web/src/main.ts"]; + expect(web, "web/src/main.ts missing from the symbol table").toBeDefined(); + const boot = (web!.functions ?? {})["boot"]; + expect(boot, "boot() missing").toBeDefined(); + expect(boot!.cfg?.length ?? 0, "nested-program callable got no CFG").toBeGreaterThan(0); + expect(Object.keys(boot!.body ?? {}).some((k) => k === "@entry")).toBe(true); + + const svcMod = app.symbol_table["web/src/app/service.ts"]; + const svc = (svcMod?.functions ?? {})["svc"]; + expect(svc?.cfg?.length ?? 0, "nested-program callee got no CFG").toBeGreaterThan(0); + + // The root program's own callables keep working — this is a widening, not a swap. + const server = (app.symbol_table["src/server.ts"]?.functions ?? {})["serve"]; + expect(server?.cfg?.length ?? 0, "root-program callable lost its CFG").toBeGreaterThan(0); + }); + test("the root program's relative import still resolves (no regression)", async () => { const app = await run(); const edge = app.call_graph.find((e) => e.source === "src/server.serve");