From 3f45cb58569ccdf33ba15862060e4abb71d3fad9 Mon Sep 17 00:00:00 2001 From: Bonobo Date: Wed, 2 Sep 2026 13:41:41 +0200 Subject: [PATCH] fix(dataflow): extract every configured program --- src/core.ts | 100 +++++++++++++++++++----------------- src/dataflow/index.ts | 3 +- test/multi-tsconfig.test.ts | 23 +++++++-- 3 files changed, 75 insertions(+), 51 deletions(-) diff --git a/src/core.ts b/src/core.ts index 48aec15..22452c7 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,6 +1,7 @@ import * as path from "node:path"; -import { buildProgramGraphs, startExtraction } from "./dataflow"; -import { type LinkerResolutions, mergeCallGraphs, runDefuseLinker, tscProvider } from "./semantic_analysis"; +import { type ExtractionHandle, buildProgramGraphs, startExtraction } from "./dataflow"; +import type { CallableGraphData } from "./dataflow/model"; +import { type CallGraphResult, type LinkerResolutions, mergeCallGraphs, runDefuseLinker, tscProvider } from "./semantic_analysis"; import { loadCache, saveCache } from "./utils"; import { materialize } from "./build"; import { inventoryArtifacts } from "./artifacts"; @@ -31,54 +32,62 @@ export async function analyze(opts: AnalysisOptions): Promise { const cached = opts.eager ? null : loadCache(cacheDir); const { project, symbol_table, programs } = buildSymbolTable(opts, mat, cached?.symbol_table ?? null, log); - // Level 3: post stage-1–4 graph extraction to the worker pool BEFORE the call-graph solve — - // extraction doesn't need callee resolution, so the two run concurrently (the contract's - // "points-to solve runs concurrently with stages 1–4") and join in buildProgramGraphs. - // - // Multi-program (#56) NOTE: each BuiltProgram carries its owning tsconfig (`configPath`), so the - // file→program config map exists here. Threading it per-file into the dataflow workers (each - // builds its own Project from ONE tsconfig, src/dataflow/worker.ts) is deferred: extraction still - // uses the root program's config, so at L3 files in a NESTED program are analyzed with the root - // options. Documented limitation — L2 call-graph resolution (the #56 gate) is fully per-program. - if (opts.analysisLevel >= 3 && programs.length > 1) { - log.warn(`L3 dataflow uses the root tsconfig for all ${programs.length} programs; nested-program files may under-resolve (see #56)`); - } - const extraction = opts.analysisLevel >= 3 ? startExtraction(project, symbol_table, mat.tsConfigFilePath, opts, log) : null; + // Each program owns both its checker and its graph extraction configuration. Start extraction + // before resolving that program's calls so worker mode preserves the intended overlap, then join + // before advancing. Reusing one pool bounds worker-project residency across monorepo programs. + const graphData = new Map(); + let graphPool: ExtractionHandle["pool"] = null; - // Call graph: the tsc resolver, per program (each with its own Project + its slice of callables - // via `only`), merged across programs. Only worth running at level >= 2: finalizeAnalysis - // discards call_graph/external_symbols/synthesized_callables at -a 1 (homeExternals/ - // homeSynthesized in src/schema/emit.ts are gated to `level >= 2`), so running the solve at - // -a 1 would compute a result that's thrown away. Levels 3/4 need it for callee resolution and - // are always >= 2, so this gate is safe. Signature gating uses the full merged symbol_table - // (passed to every program), so a cross-program in-project call resolves. - let cg: ReturnType = { edges: [], external_symbols: {}, synthesized_callables: {} }; + let cg: CallGraphResult = { edges: [], external_symbols: {}, synthesized_callables: {} }; const resolutions: LinkerResolutions = new Map(); - if (opts.analysisLevel >= 2) { - for (const prog of programs) { - const ctx = { - project: prog.project, - symbol_table, - root: opts.input, - log, - phantoms: opts.phantoms, - only: prog.fileKeys, - }; - cg = mergeCallGraphs(cg, tscProvider.build(ctx)); - // The defuse linker overlays the tsc base: it reads the callee_signature backfill the tsc - // leg just wrote, resolves what remains (tiers T1–T5, defuseLinker.ts), and returns its - // body-node resolutions out-of-band (never persisted — cache provenance rule). - const linked = runDefuseLinker(ctx); - cg = mergeCallGraphs(cg, linked.result); - for (const [caller, m] of linked.resolutions) { - const ex = resolutions.get(caller); - if (!ex) resolutions.set(caller, m); - else for (const [k, v] of m) if (!ex.has(k)) ex.set(k, v); + for (const prog of programs) { + const ownedSymbols = Object.fromEntries( + Object.entries(symbol_table).filter(([fileKey]) => prog.fileKeys.has(fileKey)), + ); + const extraction: ExtractionHandle | null = opts.analysisLevel >= 3 + ? startExtraction(prog.project, ownedSymbols, prog.configPath, opts, log, graphPool) + : null; + + try { + if (opts.analysisLevel >= 2) { + const ctx = { + project: prog.project, + symbol_table, + root: opts.input, + log, + phantoms: opts.phantoms, + only: prog.fileKeys, + }; + cg = mergeCallGraphs(cg, tscProvider.build(ctx)); + // The defuse linker overlays the tsc base: it reads the callee_signature backfill the tsc + // leg just wrote, resolves what remains (tiers T1–T5, defuseLinker.ts), and returns its + // body-node resolutions out-of-band (never persisted — cache provenance rule). + const linked = runDefuseLinker(ctx); + cg = mergeCallGraphs(cg, linked.result); + for (const [caller, m] of linked.resolutions) { + const ex = resolutions.get(caller); + if (!ex) resolutions.set(caller, m); + else for (const [k, v] of m) if (!ex.has(k)) ex.set(k, v); + } + } + + if (extraction) { + for (const [signature, data] of await extraction.promise) graphData.set(signature, data); + graphPool = extraction.pool; } + } catch (error) { + extraction?.pool?.close(); + if (graphPool !== extraction?.pool) graphPool?.close(); + throw error; } } const call_graph = cg.edges; + const extraction: ExtractionHandle | null = opts.analysisLevel >= 3 + ? { promise: Promise.resolve(graphData), pool: graphPool } + : null; + const pg = extraction ? await buildProgramGraphs(extraction, symbol_table, opts, log) : null; + // Repository-artifact layer (#101, python PR #160 parity): level-free, identical at every -a. const layer = inventoryArtifacts(opts.input, opts, symbol_table); log.info( @@ -96,9 +105,8 @@ export async function analyze(opts: AnalysisOptions): Promise { unresolved_imports: layer.unresolved_imports, }; - // Level 3 join: stages 5–7 (summary wavefront + SDG) consume the extraction AND the - // provider-backfilled callee signatures. Strictly flag-gated so -a 1/-a 2 cost nothing. - const pg = extraction ? await buildProgramGraphs(extraction, symbol_table, opts, log) : null; + // Graph extraction and the provider-backed call graph are joined above before artifacts and + // per-run schema passes, so every program contributes to the same deterministic graph envelope. // Cache the id-free base (ids/body/heritage are per-run layers stamped by finalizeAnalysis; // the cached tree must stay --app-name-free). diff --git a/src/dataflow/index.ts b/src/dataflow/index.ts index 51ded96..1bb075c 100644 --- a/src/dataflow/index.ts +++ b/src/dataflow/index.ts @@ -71,6 +71,7 @@ export function startExtraction( tsConfigFilePath: string | null, opts: AnalysisOptions, log: Logger, + existingPool: WorkerPool | null = null, ): ExtractionHandle { const callables = collectCallables(symbol_table); @@ -101,7 +102,7 @@ export function startExtraction( let pool: WorkerPool; try { - pool = new WorkerPool(workerCount); + pool = existingPool ?? new WorkerPool(workerCount); } catch (e) { log.warn(`graph workers unavailable (${(e as Error).message}); extracting sequentially`); return { promise: Promise.resolve(extractSequential(project, callables, opts)), pool: null }; diff --git a/test/multi-tsconfig.test.ts b/test/multi-tsconfig.test.ts index 2d630f9..b16b4ec 100644 --- a/test/multi-tsconfig.test.ts +++ b/test/multi-tsconfig.test.ts @@ -19,11 +19,11 @@ import * as os from "node:os"; import * as path from "node:path"; import { analyze } from "../src/core"; import type { AnalysisOptions } from "../src/options"; -import type { TSApplication } from "../src/schema"; +import { forEachCallable, type TSApplication } from "../src/schema"; const FIXTURE = path.resolve(import.meta.dir, "fixtures/multi-tsconfig-app"); -function options(): AnalysisOptions { +function options(over: Partial = {}): AnalysisOptions { return { input: FIXTURE, output: null, @@ -44,13 +44,14 @@ function options(): AnalysisOptions { phantoms: true, cacheDir: null, verbosity: 0, + ...over, }; } -async function run(): Promise { +async function run(over: Partial = {}): Promise { const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), "cants-multitsconfig-test-")); try { - return (await analyze({ ...options(), cacheDir })).internal; + return (await analyze({ ...options(over), cacheDir })).internal; } finally { fs.rmSync(cacheDir, { recursive: true, force: true }); } @@ -95,4 +96,18 @@ describe("multi-tsconfig program construction (#56)", () => { expect(edge?.target).toBe("src/util.greet"); expect(edge?.provenance).toContain("tsc"); }); + + test("level 3 emits graphs for callables owned by every program", async () => { + const app = await run({ analysisLevel: 3, graphs: ["cfg"], jobs: 2 }); + const graphed = new Set(); + for (const mod of Object.values(app.symbol_table)) { + forEachCallable(mod, (callable) => { + if (callable.cfg) graphed.add(callable.signature); + }); + } + + expect(graphed).toContain("src/server.serve"); + expect(graphed).toContain("web/src/main.boot"); + expect(graphed).toContain("web/src/app/service.svc"); + }); });