fix(dataflow): resolve each callable under its owning tsconfig

src/core.ts

@@ -35,15 +35,10 @@ export async function analyze(opts: AnalysisOptions): Promise<AnalysisResult> {
   // extraction doesn't need callee resolution, so the two run concurrently (the contract's
   // "points-to solve runs concurrently with stages 1–4") and join in buildProgramGraphs.
   //
-  // Multi-program (#56) NOTE: each BuiltProgram carries its owning tsconfig (`configPath`), so the
-  // file→program config map exists here. Threading it per-file into the dataflow workers (each
-  // builds its own Project from ONE tsconfig, src/dataflow/worker.ts) is deferred: extraction still
-  // uses the root program's config, so at L3 files in a NESTED program are analyzed with the root
-  // options. Documented limitation — L2 call-graph resolution (the #56 gate) is fully per-program.
-  if (opts.analysisLevel >= 3 && programs.length > 1) {
-    log.warn(`L3 dataflow uses the root tsconfig for all ${programs.length} programs; nested-program files may under-resolve (see #56)`);
-  }
-  const extraction = opts.analysisLevel >= 3 ? startExtraction(project, symbol_table, mat.tsConfigFilePath, opts, log) : null;
+  // Multi-program: extraction resolves each callable under the tsconfig that OWNS its file, the
+  // same assignment the symbol table and the L2 call graph use (#111). It used to index the whole
+  // repo against the root program alone, which silently skipped every callable a deeper program
+  // owned.
 
   // Call graph: the tsc resolver, per program (each with its own Project + its slice of callables
   // via `only`), merged across programs. Only worth running at level >= 2: finalizeAnalysis
@@ -77,6 +72,14 @@ export async function analyze(opts: AnalysisOptions): Promise<AnalysisResult> {
       }
     }
   }
+  // Extraction runs AFTER the solve, not concurrently with it (#112). The two only ever
+  // overlapped under `-j N > 1` -- at the default `-j 1` startExtraction evaluates
+  // extractSequential eagerly, so they were already serial. Ordering them explicitly lets
+  // extraction be the LAST reader of each program's Project, which is what makes disposal safe:
+  // on a repo with many programs, holding all of them materialised is what exhausted the heap.
+  const extraction =
+    opts.analysisLevel >= 3 ? startExtraction(programs, symbol_table, opts, log, project) : null;
+
   const call_graph = cg.edges;
 
   // Repository-artifact layer (#101, python PR #160 parity): level-free, identical at every -a.

src/dataflow/index.ts

@@ -26,7 +26,8 @@
  */
 import * as fs from "node:fs";
 import * as path from "node:path";
-import type { Project } from "ts-morph";
+import type { Node, Project } from "ts-morph";
+import type { BuiltProgram } from "../syntactic_analysis/symbolTable";
 import type { AnalysisOptions } from "../options";
 import {
   PROGRAM_GRAPHS_SCHEMA_VERSION,
@@ -66,13 +67,19 @@ export interface ExtractionHandle {
 }
 
 export function startExtraction(
-  project: Project,
+  programs: BuiltProgram[],
   symbol_table: Record<string, TSModule>,
-  tsConfigFilePath: string | null,
   opts: AnalysisOptions,
   log: Logger,
+  keepProject?: Project,
 ): ExtractionHandle {
   const callables = collectCallables(symbol_table);
+  // Every callable is extracted under the tsconfig that OWNS its file, exactly as the symbol table
+  // and the L2 call graph already resolve it (#111). Indexing the whole repo against one root
+  // program silently drops every callable a deeper program owns: `indexCallableDecls` never sees
+  // the declaration, and the `if (!fn) continue` below skips it. On vscode -- 92 programs, no root
+  // tsconfig -- that was 1,204 of 174,767 callables extracted.
+  const ownerOf = programOwnerIndex(programs, callables, opts.input);
 
   // Partition callables by owning file (round-robin over the sorted file list) so each worker
   // deeply visits only its share of the program. TSCallable.abs_path is the declaration's ABSOLUTE
@@ -95,7 +102,7 @@ export function startExtraction(
   const jobs = opts.jobs === 0 ? 1 : opts.jobs;
 
   if (jobs <= 1) {
-    return { promise: Promise.resolve(extractSequential(project, callables, opts)), pool: null };
+    return { promise: Promise.resolve(extractSequential(programs, ownerOf, callables, opts, log, keepProject)), pool: null };
   }
   const workerCount = Math.max(1, Math.min(jobs, files.length));
 
@@ -104,30 +111,44 @@ export function startExtraction(
     pool = new WorkerPool(workerCount);
   } catch (e) {
     log.warn(`graph workers unavailable (${(e as Error).message}); extracting sequentially`);
-    return { promise: Promise.resolve(extractSequential(project, callables, opts)), pool: null };
+    return { promise: Promise.resolve(extractSequential(programs, ownerOf, callables, opts, log, keepProject)), pool: null };
   }
 
-  const partitions: Array<Array<{ signature: string; path: string; absPath: string }>> = Array.from(
-    { length: workerCount },
-    () => [],
-  );
-  files.forEach((f, i) => partitions[i % workerCount]?.push(...(byFile.get(f) ?? [])));
+  // Partition WITHIN each owning program: a task's files must all share one tsconfig, because the
+  // worker builds a single Project per task config (#111). Round-robin inside a program keeps the
+  // per-worker balance the previous global round-robin had.
+  const byConfig = new Map<string, { configPath: string | null; files: string[] }>();
+  for (const f of files) {
+    const configPath = ownerOf.get(f) ?? null;
+    const key = configPath ?? "";
+    const g = byConfig.get(key) ?? { configPath, files: [] };
+    g.files.push(f);
+    byConfig.set(key, g);
+  }
+  const partitions: Array<{ configPath: string | null; sigs: Array<{ signature: string; path: string; absPath: string }> }> = [];
+  for (const key of [...byConfig.keys()].sort()) {
+    const g = byConfig.get(key)!;
+    const slots: Array<Array<{ signature: string; path: string; absPath: string }>> = Array.from(
+      { length: Math.max(1, Math.min(workerCount, g.files.length)) },
+      () => [],
+    );
+    g.files.forEach((f, i) => slots[i % slots.length]?.push(...(byFile.get(f) ?? [])));
+    for (const s of slots) if (s.length) partitions.push({ configPath: g.configPath, sigs: s });
+  }
 
   const handle: ExtractionHandle = { promise: Promise.resolve(new Map()), pool };
   handle.promise = Promise.all(
-    partitions
-      .filter((p) => p.length)
-      .map((sigs) => {
-        const task: ExtractTask = {
-          type: "extract",
-          root: opts.input,
-          tsConfigFilePath,
-          skipTests: opts.skipTests,
-          k: opts.graphFieldDepth,
-          sigs,
-        };
-        return pool.exec<CallableGraphData[]>(task);
-      }),
+    partitions.map(({ configPath, sigs }) => {
+      const task: ExtractTask = {
+        type: "extract",
+        root: opts.input,
+        tsConfigFilePath: configPath,
+        skipTests: opts.skipTests,
+        k: opts.graphFieldDepth,
+        sigs,
+      };
+      return pool.exec<CallableGraphData[]>(task);
+    }),
   )
     .then((chunks) => {
       const out = new Map<string, CallableGraphData>();
@@ -137,6 +158,7 @@ export function startExtraction(
         // main thread's — treat it as a failure, never as an empty program.
         throw new Error("workers returned no callables");
       }
+      reportCoverage(out.size, callables.size, log);
       return out;
     })
     .catch((e: Error) => {
@@ -146,28 +168,88 @@ export function startExtraction(
       log.warn(`graph extraction workers failed (${e.message}); falling back to sequential`);
       handle.pool?.close();
       handle.pool = null;
-      return extractSequential(project, callables, opts);
+      return extractSequential(programs, ownerOf, callables, opts, log, keepProject);
     });
 
   return handle;
 }
 
+/**
+ * absolute file path -> the tsconfig that OWNS it, taken from the same program assignment the
+ * symbol table used (`BuiltProgram.fileKeys`, deepest scope wins). A file no program claims maps
+ * to null, the default-options program.
+ */
+function programOwnerIndex(
+  programs: BuiltProgram[],
+  callables: Map<string, TSCallable>,
+  root: string,
+): Map<string, string | null> {
+  const byFileKey = new Map<string, string | null>();
+  for (const p of programs) for (const k of p.fileKeys) if (!byFileKey.has(k)) byFileKey.set(k, p.configPath);
+  const out = new Map<string, string | null>();
+  for (const c of callables.values()) {
+    if (out.has(c.abs_path)) continue;
+    out.set(c.abs_path, byFileKey.get(fileKeyOf(c.abs_path, root).fileKey) ?? null);
+  }
+  return out;
+}
+
 function extractSequential(
-  project: Project,
+  programs: BuiltProgram[],
+  ownerOf: Map<string, string | null>,
   callables: Map<string, TSCallable>,
   opts: AnalysisOptions,
+  log: Logger,
+  keepProject?: Project,
 ): Map<string, CallableGraphData> {
-  const astIndex = indexCallableDecls(project, opts.input);
-  const out = new Map<string, CallableGraphData>();
+  // Group first, then index ONE program at a time. Building all of them up front holds every
+  // program's declaration nodes live at once: on vscode (92 programs) that peaked at 26.9GB and
+  // the process was killed. Only one index is resident here, so the memory profile matches the
+  // single-root version while the lookup is per-owning-program (#111).
+  const byConfig = new Map<string, Array<[string, TSCallable]>>();
   for (const [sig, c] of [...callables.entries()].sort(([a], [b]) => a.localeCompare(b))) {
-    const fn = astIndex.get(sig);
-    if (!fn) continue; // bodiless (interface/abstract/ambient/implicit) or unmatchable
-    const data = extractCallableData(sig, fn, fileKeyOf(c.abs_path, opts.input).fileKey, opts.input, opts.graphFieldDepth);
-    if (data) out.set(sig, data);
+    const key = ownerOf.get(c.abs_path) ?? "";
+    const group = byConfig.get(key);
+    if (group) group.push([sig, c]);
+    else byConfig.set(key, [[sig, c]]);
   }
+
+  const out = new Map<string, CallableGraphData>();
+  for (const p of programs) {
+    const group = byConfig.get(p.configPath ?? "");
+    if (!group?.length) continue;
+    const idx = indexCallableDecls(p.project, opts.input);
+    for (const [sig, c] of group) {
+      const fn = idx.get(sig);
+      if (!fn) continue; // bodiless (interface/abstract/ambient/implicit) or unmatchable
+      const data = extractCallableData(sig, fn, fileKeyOf(c.abs_path, opts.input).fileKey, opts.input, opts.graphFieldDepth);
+      if (data) out.set(sig, data);
+    }
+    // Release this program's ASTs now that nothing else will read them (#112). Indexing a
+    // project forces tsc to parse and bind every file in it, so on a repo with many programs the
+    // materialised set is the dominant cost -- vscode has 92. `keepProject` is the root program,
+    // which finalizeAnalysis still needs for the config-use dataflow tier, so it is spared.
+    if (p.project !== keepProject) {
+      for (const sf of p.project.getSourceFiles()) p.project.removeSourceFile(sf);
+    }
+  }
+  reportCoverage(out.size, callables.size, log);
   return out;
 }
 
+/**
+ * Say how much flow was actually extracted. A near-empty L3 used to be indistinguishable from a
+ * successful one -- #111 shipped precisely because nothing reported that 0.7% of callables had
+ * been populated.
+ */
+function reportCoverage(extracted: number, collected: number, log: Logger): void {
+  if (!collected) return;
+  const pct = (100 * extracted) / collected;
+  const msg = `dataflow: extracted ${extracted.toLocaleString()} of ${collected.toLocaleString()} callables (${pct.toFixed(1)}%)`;
+  if (pct < 50) log.warn(`${msg} — most callables produced no control/data flow`);
+  else log.info(msg);
+}
+
 // ------------------------------------------------------------------------------------------------
 // Stages 5–7 + emission
 // ------------------------------------------------------------------------------------------------

src/utils/serialize.ts

@@ -23,7 +23,68 @@ export async function emit(application: TSAnalysis, opts: AnalysisOptions): Prom
     return;
   }
   fs.mkdirSync(opts.output, { recursive: true });
-  fs.writeFileSync(path.join(opts.output, "analysis.json"), JSON.stringify(application));
+  writeAnalysisJson(path.join(opts.output, "analysis.json"), application);
+}
+
+/**
+ * Write the envelope WITHOUT ever holding it as one string (#112).
+ *
+ * `JSON.stringify(application)` materialises the entire output before a byte is written, so peak
+ * memory carries the tree AND its serialisation at once, and a large enough analysis exceeds the
+ * runtime's maximum string length outright: vscode at -a 4 dies with `RangeError: Out of memory`
+ * in stringify after the analysis itself has completed successfully.
+ *
+ * The envelope's shape is fixed and its two unbounded members are `symbol_table` (keyed by module)
+ * and the application-scope edge arrays, so both are streamed element by element. Each element is
+ * still stringified individually — bounded by the largest single module, not by the whole repo.
+ * Output is byte-identical to the previous whole-string write.
+ */
+function writeAnalysisJson(file: string, envelope: TSAnalysis): void {
+  const fd = fs.openSync(file, "w");
+  const put = (chunk: string): void => {
+    fs.writeSync(fd, chunk);
+  };
+  try {
+    const { application: app, ...head } = envelope as unknown as Record<string, unknown>;
+    const root = app as Record<string, unknown>;
+
+    // envelope head — every key except `application`
+    put("{");
+    for (const [k, v] of Object.entries(head)) put(`${JSON.stringify(k)}:${JSON.stringify(v)},`);
+    put(`"application":{`);
+
+    // Walk the application's keys in INSERTION ORDER so the bytes match what JSON.stringify
+    // produced; only the two unbounded members stream, everything else is small.
+    let firstKey = true;
+    for (const [k, v] of Object.entries(root)) {
+      if (v === undefined) continue;
+      if (!firstKey) put(",");
+      firstKey = false;
+      put(`${JSON.stringify(k)}:`);
+      if (k === "symbol_table" && v && typeof v === "object") {
+        put("{");
+        let first = true;
+        for (const [key, mod] of Object.entries(v as Record<string, unknown>)) {
+          if (!first) put(",");
+          first = false;
+          put(`${JSON.stringify(key)}:${JSON.stringify(mod)}`);
+        }
+        put("}");
+      } else if (Array.isArray(v)) {
+        put("[");
+        for (let i = 0; i < v.length; i++) {
+          if (i) put(",");
+          put(JSON.stringify(v[i]));
+        }
+        put("]");
+      } else {
+        put(JSON.stringify(v));
+      }
+    }
+    put("}}");
+  } finally {
+    fs.closeSync(fd);
+  }
 }
 
 /**

test/multi-tsconfig.test.ts

@@ -56,6 +56,16 @@ async function run(): Promise<TSApplication> {
   }
 }
 
+async function runLevel3(): Promise<TSApplication> {
+  const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), "cants-multitsconfig-l3-"));
+  try {
+    const r = await analyze({ ...options(), analysisLevel: 3, graphs: ["cfg", "dfg", "pdg"], cacheDir });
+    return r.application.application;
+  } finally {
+    fs.rmSync(cacheDir, { recursive: true, force: true });
+  }
+}
+
 describe("multi-tsconfig program construction (#56)", () => {
   test("both programs' files are in one merged symbol table, each owned once", async () => {
     const app = await run();
@@ -89,6 +99,30 @@ describe("multi-tsconfig program construction (#56)", () => {
     expect(Object.keys(app.external_symbols ?? {})).not.toContain("@app/service.svc");
   });
 
+  // #111: extraction used to index every callable against the ROOT program alone. A file a DEEPER
+  // program owns is absent from that index, so `if (!fn) continue` skipped it silently — the run
+  // succeeded and produced no flow. Measured on vscode (92 programs, no root tsconfig): 1,204 of
+  // 174,767 callables. Here the root tsconfig excludes `web`, so every web callable is owned by
+  // the nested program and is exactly the case that vanished.
+  test("L3 populates callables owned by the NESTED program, not just the root's (#111)", async () => {
+    const app = await runLevel3();
+
+    const web = app.symbol_table["web/src/main.ts"];
+    expect(web, "web/src/main.ts missing from the symbol table").toBeDefined();
+    const boot = (web!.functions ?? {})["boot"];
+    expect(boot, "boot() missing").toBeDefined();
+    expect(boot!.cfg?.length ?? 0, "nested-program callable got no CFG").toBeGreaterThan(0);
+    expect(Object.keys(boot!.body ?? {}).some((k) => k === "@entry")).toBe(true);
+
+    const svcMod = app.symbol_table["web/src/app/service.ts"];
+    const svc = (svcMod?.functions ?? {})["svc"];
+    expect(svc?.cfg?.length ?? 0, "nested-program callee got no CFG").toBeGreaterThan(0);
+
+    // The root program's own callables keep working — this is a widening, not a swap.
+    const server = (app.symbol_table["src/server.ts"]?.functions ?? {})["serve"];
+    expect(server?.cfg?.length ?? 0, "root-program callable lost its CFG").toBeGreaterThan(0);
+  });
+
   test("the root program's relative import still resolves (no regression)", async () => {
     const app = await run();
     const edge = app.call_graph.find((e) => e.source === "src/server.serve");