diff --git a/src/cli.ts b/src/cli.ts index dfce972..7602e04 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -59,6 +59,13 @@ export function buildProgram(): Command { .option("--no-phantoms", "disable phantom (external) nodes for imported/required library calls") .option("--resolve-installed", "probe node_modules metadata for import→package binding (default: repo files only)") .option("--no-artifact-text", "keep the artifact inventory but drop captured raw text") + .option( + "--program ", + "restrict the run to these programs, named by scope dir relative to --input ('' for the root program); repeatable", + ) + .option("--list-programs", "list the discovered programs, one per line, and exit") + .option("--emit-ir", "persist this shard's graph IR for a later cross-shard stitch") + .option("--no-repo-sections", "skip artifacts/dependencies/unresolved_imports (repo-scoped; compute them once per repository, not once per shard)") .option("-c, --cache-dir ", "cache/intermediate directory") .option("-v, --verbose", "increase verbosity (repeatable)", (_v: string, prev: number) => prev + 1, 0) .allowExcessArguments(true); @@ -147,6 +154,11 @@ export function parseArgs(argv: string[]): AnalysisOptions { graphFieldDepth: k, jobs, targetFiles: targets, + programFilter: + Array.isArray(o.program) && o.program.length ? o.program.map(String) : null, + listPrograms: Boolean(o.listPrograms), + emitIr: Boolean(o.emitIr), + noRepoSections: o.repoSections === false, skipTests: o.includeTests ? false : true, eager: Boolean(o.eager), // commander maps --no-build / --no-phantoms to opts.build/phantoms === false diff --git a/src/core.ts b/src/core.ts index cc12435..907b443 100644 --- a/src/core.ts +++ b/src/core.ts @@ -7,7 +7,7 @@ import { inventoryArtifacts } from "./artifacts"; import type { AnalysisOptions } from "./options"; import type { AnalysisInternal } from "./schema"; import { type AnalysisResult, finalizeAnalysis } from "./schema/emit"; -import { buildSymbolTable } from "./syntactic_analysis"; +import { buildSymbolTable, programName } from "./syntactic_analysis"; import { Logger } from "./utils"; import { checkerFailures, resetCheckerFailures } from "./schema/checker"; @@ -19,6 +19,15 @@ export type { AnalysisResult } from "./schema/emit"; * run the per-run pass spine (ids / body / heritage / homing / callees / attach) and assemble * the wire envelope. Returns BOTH views: the wire `application` and the live `internal` tree. */ +/** + * The programs this input discovers, deepest scope first — the names `--program` accepts (#146). + * Shard enumeration for an orchestrator: it must be able to find the shards before running them. + */ +export function discoverPrograms(opts: AnalysisOptions): string[] { + const log = new Logger(opts.verbosity); + return materialize(opts, log).programs.map((spec) => programName(spec, opts.input)); +} + export async function analyze(opts: AnalysisOptions): Promise { const log = new Logger(opts.verbosity); log.info(`analyzing ${opts.input} (level ${opts.analysisLevel})`); @@ -26,6 +35,7 @@ export async function analyze(opts: AnalysisOptions): Promise { const cacheDir = opts.cacheDir ?? path.join(opts.input, ".codeanalyzer"); const mat = materialize(opts, log); + for (const note of mat.notes) log.debug(note); const cached = opts.eager ? null : loadCache(cacheDir); @@ -83,11 +93,19 @@ export async function analyze(opts: AnalysisOptions): Promise { const call_graph = cg.edges; // Repository-artifact layer (#101, python PR #160 parity): level-free, identical at every -a. - const layer = inventoryArtifacts(opts.input, opts, symbol_table); - log.info( - `artifacts: ${Object.keys(layer.artifacts).length} files, ${layer.dependencies.length} dependency records, ` + - `${layer.unresolved_imports.length} unresolved imports`, - ); + // Repo-SCOPED, not program-scoped: it is derived from --input, so a `--program` shard would + // recompute the whole repository's inventory (#146 measured 4,961 artifacts and 4,107 dependency + // records for a shard whose code analysis covers 6,758 modules). Under sharding it is computed + // once, by one run, and every other shard passes --no-repo-sections. + const layer = opts.noRepoSections + ? { artifacts: {}, dependencies: [], unresolved_imports: [] } + : inventoryArtifacts(opts.input, opts, symbol_table); + if (opts.noRepoSections) log.info("artifacts: skipped (--no-repo-sections)"); + else + log.info( + `artifacts: ${Object.keys(layer.artifacts).length} files, ${layer.dependencies.length} dependency records, ` + + `${layer.unresolved_imports.length} unresolved imports`, + ); const app: AnalysisInternal = { symbol_table, diff --git a/src/dataflow/index.ts b/src/dataflow/index.ts index ac599df..a35e7ef 100644 --- a/src/dataflow/index.ts +++ b/src/dataflow/index.ts @@ -29,6 +29,7 @@ import * as path from "node:path"; import type { Node, Project } from "ts-morph"; import type { BuiltProgram } from "../syntactic_analysis/symbolTable"; import type { AnalysisOptions } from "../options"; +import { writeIr } from "./ir"; import { PROGRAM_GRAPHS_SCHEMA_VERSION, fileKeyOf, @@ -316,6 +317,8 @@ export async function buildProgramGraphs( const sdg_edges = wantSdg ? assembleSdg(datas, callSites, summaries) : []; persistSummaries(opts, symbol_table, callables, summaries, log); + // Wave-1 shard IR (#112 step 4): everything the cross-shard stitch needs and nothing tsc owns. + if (opts.emitIr) writeIr(opts, opts.programFilter ?? [""], datas, callSites, summaries, log); return { schema_version: PROGRAM_GRAPHS_SCHEMA_VERSION, k_limit: opts.graphFieldDepth, functions, sdg_edges }; } finally { diff --git a/src/dataflow/ir.ts b/src/dataflow/ir.ts new file mode 100644 index 0000000..e681176 --- /dev/null +++ b/src/dataflow/ir.ts @@ -0,0 +1,146 @@ +/** + * Shard IR persistence (#112 step 4, unit 2) — what wave 2 reads. + * + * The two-wave design rests on one fact: the cross-shard interprocedural stitch needs NO tsc state. + * On vscode/src at `-a 4`, 21.33 GB is already committed before the interprocedural phase starts, + * while that phase's entire retained state is 0.48 GB (`datas` 0.34, `ddg` 0.09, `summaries` 0.05). + * So if wave 1 writes its graph IR down, wave 2 can redo the cross-shard fixpoint over ~1 GB of + * data instead of paying the 21 GB parse/bind/check cost again. + * + * Written as NDJSON, one record per line, deliberately: `JSON.stringify` on the whole IR would + * build a single multi-hundred-megabyte string — the same emit-time wall #112 lists as ceiling 3. + * NDJSON streams out and streams back in, and a reader can skip record kinds it does not need. + * + * The header pins every input that would silently corrupt a union if it differed between shards: + * `can://` ids embed `--input` and `--app-name`, and summaries are only comparable at one + * `k_limit`. A mismatch is an error at load, never a merge. + */ +import * as fs from "node:fs"; +import * as path from "node:path"; +import type { CallableGraphData } from "./model"; +import type { CallSiteRef, FunctionSummary } from "./summaries"; +import type { AnalysisOptions } from "../options"; +import type { Logger } from "../utils"; +import { PROGRAM_GRAPHS_SCHEMA_VERSION } from "../schema/graphs"; + +export const IR_FILENAME = "graphs_ir.ndjson"; + +/** Identity of the run that produced a shard's IR. Every field must match across shards. */ +export interface IrHeader { + kind: "header"; + ir_version: string; + schema_version: string; + k_limit: number; + app_name: string; + /** Absolute input root — `can://` file keys are relative to it, so shards must share one. */ + input: string; + /** Programs this shard analysed (scope names, as `--program` takes them). */ + programs: string[]; +} + +/** + * IR record version. Separate from PROGRAM_GRAPHS_SCHEMA_VERSION because this file now has a + * READER and therefore a cross-run contract of its own: the wire shape of `CallableGraphData` can + * change without the emitted program-graphs schema changing, and vice versa. + */ +export const IR_VERSION = "1.0.0"; + +export type IrRecord = + | IrHeader + | { kind: "callable"; sig: string; data: CallableGraphData } + | { kind: "callsites"; sig: string; sites: CallSiteRef[] } + | { kind: "summary"; sig: string; summary: FunctionSummary }; + +export function irPath(opts: AnalysisOptions): string { + return path.join(opts.cacheDir ?? path.join(opts.input, ".codeanalyzer"), IR_FILENAME); +} + +/** + * Write this shard's IR. Streams record-by-record through an fd rather than joining, so peak stays + * flat regardless of callable count. + */ +export function writeIr( + opts: AnalysisOptions, + programs: string[], + datas: Map, + callSites: Map, + summaries: Map, + log: Logger, +): void { + const file = irPath(opts); + fs.mkdirSync(path.dirname(file), { recursive: true }); + const fd = fs.openSync(file, "w"); + try { + const write = (r: IrRecord): void => { + fs.writeSync(fd, `${JSON.stringify(r)}\n`); + }; + write({ + kind: "header", + ir_version: IR_VERSION, + schema_version: PROGRAM_GRAPHS_SCHEMA_VERSION, + k_limit: opts.graphFieldDepth, + app_name: opts.appName ?? "", + input: opts.input, + programs, + }); + // Sorted so a shard's IR is byte-reproducible across runs (the equivalence test compares files). + for (const sig of [...datas.keys()].sort()) write({ kind: "callable", sig, data: datas.get(sig) as CallableGraphData }); + for (const sig of [...callSites.keys()].sort()) write({ kind: "callsites", sig, sites: callSites.get(sig) as CallSiteRef[] }); + for (const sig of [...summaries.keys()].sort()) write({ kind: "summary", sig, summary: summaries.get(sig) as FunctionSummary }); + log.info(`ir: wrote ${datas.size} callables to ${path.basename(file)}`); + } finally { + fs.closeSync(fd); + } +} + +export interface LoadedIr { + header: IrHeader; + datas: Map; + callSites: Map; + summaries: Map; +} + +/** Read one shard's IR. Line-by-line, so a large shard never becomes one string. */ +export function readIr(file: string): LoadedIr { + const datas = new Map(); + const callSites = new Map(); + const summaries = new Map(); + let header: IrHeader | null = null; + + // Split on newlines from a single read: records are individually small, and Bun has no + // synchronous line reader. The file is IR, not output — if it ever outgrows this, the reader + // becomes a stream without changing the format. + for (const line of fs.readFileSync(file, "utf8").split("\n")) { + if (!line) continue; + const rec = JSON.parse(line) as IrRecord; + if (rec.kind === "header") header = rec; + else if (rec.kind === "callable") datas.set(rec.sig, rec.data); + else if (rec.kind === "callsites") callSites.set(rec.sig, rec.sites); + else if (rec.kind === "summary") summaries.set(rec.sig, rec.summary); + } + if (!header) throw new Error(`${file}: no IR header record`); + if (header.ir_version !== IR_VERSION) { + throw new Error(`${file}: IR version ${header.ir_version}, expected ${IR_VERSION} — re-run wave 1`); + } + return { header, datas, callSites, summaries }; +} + +/** + * Every shard must come from one logical run. `can://` ids embed the input root and app name, and + * summaries only compose at a single `k_limit`, so a divergence here does not produce a partial + * union — it produces a WRONG one, silently. Hence an error rather than a warning. + */ +export function assertCompatible(shards: LoadedIr[]): void { + const [first, ...rest] = shards; + if (!first) throw new Error("no shard IR to stitch"); + for (const s of rest) { + for (const k of ["ir_version", "schema_version", "k_limit", "app_name", "input"] as const) { + if (s.header[k] !== first.header[k]) { + throw new Error( + `shard mismatch on ${k}: ${JSON.stringify(first.header[k])} vs ${JSON.stringify(s.header[k])} — ` + + `every shard must share --input, --app-name and --graph-field-depth`, + ); + } + } + } +} diff --git a/src/index.ts b/src/index.ts index 3b6de23..adeea18 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ #!/usr/bin/env node -import { analyze } from "./core"; +import { analyze, discoverPrograms } from "./core"; import { parseArgs } from "./cli"; import { emit, emitSchema } from "./utils"; @@ -11,6 +11,10 @@ async function main(): Promise { emitSchema(opts); return; } + if (opts.listPrograms) { + for (const name of discoverPrograms(opts)) process.stdout.write(`${name}\n`); + return; + } const result = await analyze(opts); await emit(result.application, opts); } catch (e) { diff --git a/src/options/options.ts b/src/options/options.ts index 2ee9c81..fbf111b 100644 --- a/src/options/options.ts +++ b/src/options/options.ts @@ -38,6 +38,26 @@ export interface AnalysisOptions { jobs: number; /** Restrict analysis to these files (project-relative or absolute). null ⇒ whole project. */ targetFiles: string[] | null; + /** + * Restrict analysis to these PROGRAMS (#146) — each named by its SCOPE dir relative to + * `input` (`` for the input's own program). null ⇒ every program. Ownership is still computed against ALL discovered programs and filtered afterwards, + * so a file owned by a deeper unselected tsconfig is excluded rather than reassigned. + */ + programFilter: string[] | null; + /** Print the discovered programs (one per line) and exit, for shard orchestration. */ + listPrograms?: boolean; + /** Persist this shard's graph IR (`graphs_ir.ndjson`) so a later wave-2 stitch can read it. */ + emitIr?: boolean; + /** + * Skip the repository-artifact layer (artifacts/dependencies/unresolved_imports). + * + * Those sections are repo-scoped and level-free: they are derived from `--input`, not from the + * analysed programs, so a `--program` shard recomputes ALL of them. On vscode that is the + * difference between 28.75 GB and 39.16 GB for one shard, and under a full shard run it would + * be paid 92 times over for a result that is identical every time. An orchestrator computes + * them once and passes this on every other shard. + */ + noRepoSections?: boolean; /** Skip test trees (default true). */ skipTests: boolean; /** Force a clean rebuild instead of reusing the cache. */ diff --git a/src/syntactic_analysis/symbolTable.ts b/src/syntactic_analysis/symbolTable.ts index a0a2dc9..f6296f2 100644 --- a/src/syntactic_analysis/symbolTable.ts +++ b/src/syntactic_analysis/symbolTable.ts @@ -43,6 +43,41 @@ function ownerProgram(absPath: string, programs: ProgramSpec[]): ProgramSpec { return programs[programs.length - 1]!; // root program is a universal ancestor; unreachable fallback } +/** + * A program's stable CLI name (#146). + * + * The name is the program's SCOPE directory, not its tsconfig: scope is what decides which files + * the program owns (`ownerProgram` matches on `scopeDir`), and a nested `tsconfig.json` that only + * `references` others resolves to its LEAF config — so `web/tsconfig.json` becomes a program named + * for scope `web` whose configPath is `web/src/tsconfig.app.json`. Two specs can even share one + * leaf config under different scopes, which is why the config path alone is not a usable identity. + * + * `` names the input root's own program. + */ +export function programName(spec: ProgramSpec, root: string): string { + const rel = path.relative(root, spec.scopeDir).split(path.sep).join("/"); + return rel === "" ? "" : rel; +} + +/** + * Which programs this run analyses. `null` filter ⇒ all of them. + * + * The filter selects programs; it must NEVER change how files are assigned to them. Ownership is + * computed against the FULL spec list and filtered afterwards (see buildSymbolTable), because + * `ownerProgram` falls back to the root program: filtering the list first would pull files owned by + * a deeper, unselected tsconfig into a selected ancestor and compile them under the wrong config. + */ +export function selectPrograms(specs: ProgramSpec[], root: string, filter: string[] | null): ProgramSpec[] { + if (!filter) return specs; + const want = new Set( + filter.map((f) => { + const n = f.split(path.sep).join("/").replace(/^\.\//, "").replace(/\/+$/, ""); + return n === "" || n === "." ? "" : n; + }), + ); + return specs.filter((s) => want.has(programName(s, root))); +} + export function buildSymbolTable( opts: AnalysisOptions, mat: Materialization, @@ -60,13 +95,26 @@ export function buildSymbolTable( // Assign every discovered file to exactly one program (deepest scope wins), then construct one // Project per program from ONLY its files — so each file resolves under the tsconfig that governs // it (module resolution, `paths` aliases, lib) instead of a single root program swallowing all. + // Ownership FIRST, against every discovered program, then filter (#146). Doing it the other way + // round would reassign a deeper program's files to a selected ancestor -- see selectPrograms. const assignment = new Map(); for (const s of specs) assignment.set(s, []); for (const f of allProjectFiles) assignment.get(ownerProgram(f.absPath, specs))!.push(f); + const selected = selectPrograms(specs, root, opts.programFilter); + if (opts.programFilter && selected.length === 0) { + // Hard error, not a warning: an orchestrator typo must not silently produce an empty shard + // that then unions cleanly into a graph missing a third of the repository. + throw new Error( + `no program matched --program ${opts.programFilter.join(", ")}. ` + + `Discovered: ${specs.map((x) => programName(x, root)).join(", ")}`, + ); + } + const selectedSet = new Set(selected); + const projectOf = new Map(); const programs: BuiltProgram[] = []; - for (const s of specs) { + for (const s of selected) { const project = createProject(s.configPath); const files = assignment.get(s)!; const fileKeys = new Set(); @@ -87,6 +135,9 @@ export function buildSymbolTable( let built = 0; let fromCache = 0; for (const f of buildFiles) { + // A file owned by an unselected program is EXCLUDED, never reassigned -- including on the + // cache path, or a warm cache would smuggle other shards' modules back into the output. + if (!selectedSet.has(ownerProgram(f.absPath, specs))) continue; if (cached && !opts.eager && cached[f.fileKey] && fileUnchanged(f.absPath, cached[f.fileKey])) { symbol_table[f.fileKey] = cached[f.fileKey]; fromCache++; @@ -105,8 +156,10 @@ export function buildSymbolTable( log.info(`symbol table: ${built} built, ${fromCache} cached, ${Object.keys(symbol_table).length} modules`); // The root program is always last; its Project is the one legacy single-program consumers expect. - const rootProject = projectOf.get(specs[specs.length - 1]!)!; - return { project: rootProject, symbol_table, files: buildFiles, programs }; + // Under --program the root may not be selected, so fall back to the shallowest SELECTED program + // (the list is deepest-first, so that is its last entry). + const rootProject = projectOf.get(selected[selected.length - 1]!)!; + return { project: rootProject, symbol_table, files: buildFiles.filter((f) => selectedSet.has(ownerProgram(f.absPath, specs))), programs }; } /** The fallback compiler options when the target has no tsconfig (shared with graph workers). */ diff --git a/test/program-selector.test.ts b/test/program-selector.test.ts new file mode 100644 index 0000000..eab83e9 --- /dev/null +++ b/test/program-selector.test.ts @@ -0,0 +1,73 @@ +/** + * `--program` shard selection (#146), unit 1 of the sharded two-wave L4 design. + * + * The load-bearing rule: selection must NOT change file→program assignment. `ownerProgram` falls + * back to the ROOT program for any file no scope contains, so filtering the spec list *before* + * assignment would hand a selected ancestor every file its deeper, unselected descendants own — + * compiling them under the wrong tsconfig (wrong module resolution, wrong `paths`, wrong lib). + * Ownership is computed globally and filtered afterwards. The second test is what catches a + * regression to the other order. + * + * Programs are named by SCOPE, not by tsconfig path: `web/tsconfig.json` here only `references` + * others, so it resolves to the leaf `web/src/tsconfig.app.json` under scope `web`, and two specs + * can share one leaf config under different scopes. + */ +import { describe, expect, test } from "bun:test"; +import * as path from "node:path"; +import { analyze, discoverPrograms } from "../src/core"; +import type { AnalysisOptions } from "../src/options"; + +const APP = path.resolve("test/fixtures/multi-tsconfig-app"); +const base = { input: APP, appName: "m", analysisLevel: 2, noBuild: true, emit: "json" }; +const opts = (programFilter: string[] | null) => ({ ...base, programFilter } as unknown as AnalysisOptions); + +type Envelope = { application: { symbol_table: Record } }; +const tableOf = (a: unknown) => (a as Envelope).application.symbol_table; +const modulesOf = (a: unknown) => Object.keys(tableOf(a)).sort(); + +describe("--program shard selection", () => { + test("enumerates programs by scope, deepest first", () => { + // deepest-first is the ordering ownerProgram depends on: first containing scope wins. + expect(discoverPrograms(opts(null))).toEqual(["web", ""]); + }); + + test("selecting an ancestor does NOT absorb a deeper program's files", async () => { + // web/src/** is owned by scope `web`. Selecting only the root must leave those files OUT, + // not compile them under the root tsconfig. Filter-then-assign would include both. + const res = await analyze(opts([""])); + expect(modulesOf(res.application)).toEqual(["src/server.ts", "src/util.ts"]); + }); + + test("selecting a program yields exactly the files it owns", async () => { + const res = await analyze(opts(["web"])); + expect(modulesOf(res.application)).toEqual(["web/src/app/service.ts", "web/src/main.ts"]); + }); + + test("selecting every program reproduces the unsharded module set and ids exactly", async () => { + const whole = await analyze(opts(null)); + const sharded = await analyze(opts(discoverPrograms(opts(null)))); + expect(modulesOf(sharded.application)).toEqual(modulesOf(whole.application)); + // ids embed --input and --app-name, never the program, so they must be byte-identical + const ids = (a: unknown) => Object.values(tableOf(a)).map((m) => m.id).sort(); + expect(ids(sharded.application)).toEqual(ids(whole.application)); + }); + + test("the shards are disjoint and their union is the whole", async () => { + const perShard: string[][] = []; + for (const name of discoverPrograms(opts(null))) perShard.push(modulesOf((await analyze(opts([name]))).application)); + const flat = perShard.flat(); + expect(new Set(flat).size).toBe(flat.length); // ownerProgram assigns exactly one owner per file + expect(flat.sort()).toEqual(modulesOf((await analyze(opts(null))).application)); + }); + + test("an unmatched --program is a hard error, not a silently empty shard", async () => { + await expect(analyze(opts(["nope"]))).rejects.toThrow(/no program matched/); + }); + + test("trailing slashes and './' spellings of a scope still match", async () => { + expect(modulesOf((await analyze(opts(["./web/"]))).application)).toEqual([ + "web/src/app/service.ts", + "web/src/main.ts", + ]); + }); +});