Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/ask/ask.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1447,13 +1447,20 @@ export function skeleton(dir: string, file: string, opts: { contextDir?: string

let defs = graph.nodes.filter((n) => n.kind !== "file" && n.path === file);
if (!defs.length) {
// #816: "no definitions indexed" is a lie for a file the build never
// parsed — name the failure instead. Same path/basename resolution as the
// defs match above.
const matches = new Set(
graph.nodes.filter((n) => n.path === file || n.path.endsWith(`/${file}`)).map((n) => n.path),
);
if (matches.size > 1)
return { file, entries: [], note: `ambiguous — matches: ${[...matches].sort().join(", ")}` };
const [path] = matches;
if (path) defs = graph.nodes.filter((n) => n.kind !== "file" && n.path === path);
const failed = (path ?? file) === file
? graph.meta.parseFailed?.find((f) => f.path === file || f.path.endsWith(`/${file}`))
: graph.meta.parseFailed?.find((f) => f.path === path);
if (failed) return { file, entries: [], note: `this file failed to parse at build time, so nothing was indexed — ${failed.error}` };
}
if (!defs.length) return { file, entries: [], note: "no definitions indexed for this file" };

Expand Down
8 changes: 8 additions & 0 deletions src/graph/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,13 @@ export async function buildGraph(
// assembled, so the merge-tiny-scopes-into-root guard runs here.
const scopes = applyMinSubstanceGuard(discoveredScopes, nodes);

// #816: parse failures ride the meta, so the search answers can say "these
// were never searched" instead of blaming the pattern. Error entries here
// cover fresh throws AND cached failures re-parsed on this run (#312).
const parseFailed = Object.entries(entries).flatMap(([path, entry]) =>
entry.error ? [{ path, error: entry.error }] : [],
);

// Assemble the graph BEFORE the meaning pass, so the crux pass can checkpoint it to
// disk periodically (#128): crux/summary mutate node objects in place and never
// change the node/edge SET, so `meta` stays valid; the opt-in LSP pass below is the
Expand All @@ -383,6 +390,7 @@ export async function buildGraph(
languages: [...langs].sort(),
extensions,
scopes,
...(parseFailed.length ? { parseFailed } : {}),
},
nodes,
edges,
Expand Down
3 changes: 3 additions & 0 deletions src/graph/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,9 @@ export interface GraphV1 {
/** Ranking scopes: posix path prefixes relative to the graph root, "" = root scope.
* Absent (old graphs) ≡ [{ prefix: "", label: "" }]. Sorted by prefix length desc. */
scopes?: ScopeV1[];
/** Files that failed to parse at build time and were never searched (#816) —
* fresh failures and cached-abort re-parses both. Absent (old graphs) ≡ unknown. */
parseFailed?: { path: string; error: string }[];
};
nodes: NodeV1[];
edges: EdgeV1[];
Expand Down
4 changes: 3 additions & 1 deletion src/graph/workspace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,7 @@ export function federateGrep(
let filesSearched = 0;
let totalHits = 0;
const truncated = { files: 0, hits: 0 };
let parseFailed = 0;
let savedFiles = 0;
let savedChars = 0;

Expand All @@ -584,6 +585,7 @@ export function federateGrep(
totalHits += r.totalHits;
truncated.files += r.truncated.files;
truncated.hits += r.truncated.hits;
parseFailed += r.parseFailed;
if (r.saved) {
savedFiles += r.saved.files;
savedChars += r.saved.baselineChars;
Expand All @@ -599,7 +601,7 @@ export function federateGrep(

groups.sort((a, b) => b.inDegree - a.inDegree || a.path.localeCompare(b.path));
const saved: Savings | undefined = savedChars > 0 ? { files: savedFiles, baselineChars: savedChars } : undefined;
const result: GrepResult = { pattern, filesSearched, totalHits, groups, truncated, saved };
const result: GrepResult = { pattern, filesSearched, totalHits, groups, truncated, parseFailed, saved };
return { result, coverage: coverageNote(wg) };
}

Expand Down
22 changes: 17 additions & 5 deletions src/search/grep-cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,15 @@ function formatGroup(g: GrepGroup): string {
return lines.join("\n");
}

/** `"<pattern>" — N hits in M symbols across K files (searched T indexed files)` */
/** `"<pattern>" — N hits in M symbols across K files (searched T indexed files)`; when the graph
* holds parse failures, the files that were never searched are named in the same breath (#816) —
* a hit count is not "all usages" if a third of the repo never made it into the index. */
export function formatGrepHeader(result: GrepResult): string {
const filesHit = new Set(result.groups.map((g) => g.path)).size;
return `"${result.pattern}" — ${result.totalHits} hits in ${result.groups.length} symbols across ${filesHit} files (searched ${result.filesSearched} indexed files)`;
const parseFailed = result.parseFailed > 0
? `; ${result.parseFailed} source file${result.parseFailed === 1 ? "" : "s"} failed to parse and were not searched (run ${cmd("graft build")} --no-reuse, or use rg for exhaustive search)`
: "";
return `"${result.pattern}" — ${result.totalHits} hits in ${result.groups.length} symbols across ${filesHit} files (searched ${result.filesSearched} indexed files${parseFailed})`;
}

/** Loud truncation note — dropped counts are never silent. */
Expand Down Expand Up @@ -77,9 +82,16 @@ export function formatGrepResult(result: GrepResult): string {
* is some, the note names it instead of saying all the code was searched. */
export function zeroHitNote(result: GrepResult, ignored: string[] = []): string {
const retry = `no hits for "${result.pattern}" in ${result.filesSearched} indexed files. The pattern may be too specific — retry ${cmd("graft grep")} with a bare symbol name or short substring (drop the receiver, full signature, and regex anchors).`;
const base = ignored.length
? `${retry} Git-ignored code isn't indexed and wasn't searched: ${ignoredLabel(ignored)}. Run grep -rn there, and on docs and configs`
: `${retry} All indexed code was searched; use raw grep -rn only for genuinely unindexed files (docs, configs, brand-new files)`;
const parseFailed = result.parseFailed > 0
? `${result.parseFailed} source file${result.parseFailed === 1 ? "" : "s"} failed to parse and were not searched (run ${cmd("graft build")} --no-reuse, or use rg for exhaustive search)`
: null;
// "All indexed code was searched" is a lie while parse failures ride the
// graph, so the failure note replaces it, not joins it (#816).
const base = parseFailed
? `${retry} ${parseFailed}`
: ignored.length
? `${retry} Git-ignored code isn't indexed and wasn't searched: ${ignoredLabel(ignored)}. Run grep -rn there, and on docs and configs`
: `${retry} All indexed code was searched; use raw grep -rn only for genuinely unindexed files (docs, configs, brand-new files)`;
const { files } = result.truncated;
if (files === 0) return base;
return `${base} — note: ${files} indexed file${files === 1 ? "" : "s"} could not be read (stale graph? run ${cmd("graft build")})`;
Expand Down
4 changes: 4 additions & 0 deletions src/search/grep.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ export interface GrepResult {
* couldn't be read from disk. `hits`: matches found beyond `maxHits`,
* counted but not collected. */
truncated: { files: number; hits: number };
/** Source files that failed to parse at build time and were never
* searched (#816). 0 when the graph predates `meta.parseFailed`. */
parseFailed: number;
/** Tokens-saved baseline: the files that had hits, read whole. Undefined
* when there were no hits or the graph predates file sizing. */
saved?: Savings;
Expand Down Expand Up @@ -216,6 +219,7 @@ export function grepGraph(graph: GraphV1, repoRoot: string, pattern: string, opt
totalHits: collected,
groups: sortedGroups,
truncated: { files: truncatedFiles, hits: truncatedHits },
parseFailed: graph.meta.parseFailed?.length ?? 0,
saved: savingsFor(graph, sortedGroups.map((g) => g.path)),
};
}
38 changes: 38 additions & 0 deletions test/ask.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import { tmpdir } from "node:os";
import { readFileSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { buildGraph } from "../src/graph/build.js";
import { readGraph, writeGraph, wiringPath } from "../src/graph/write.js";
import { contextDirFor } from "../src/context/node-file.js";
import { ask, formatAsk, skeleton, formatSkeleton, isTestPath } from "../src/ask/ask.js";

test("isTestPath: de-ranks test files, not real source", () => {
Expand Down Expand Up @@ -186,6 +188,42 @@ test("skeleton lists a file's definitions in span order, matches by basename", a
}
});

// Issue #816: a file that failed to parse must say so, not pretend it has no
// definitions. A file with no failure keeps the old note.
test("skeleton blames the parse failure for a failed file, with the build error (#816)", async () => {
const dir = mkdtempSync(join(tmpdir(), "graft-skel-failed-"));
try {
writeFileSync(
join(dir, "api.ts"),
`export function first(a: number): number {\n return a;\n}\n`,
);
await buildGraph(dir);
const outDir = contextDirFor(dir);
const g = readGraph(wiringPath(outDir));
assert.ok(g, "graph built");
g.meta.parseFailed = [
{ path: "broken.cs", error: "broken.cs: parse failed — c_sharp grammar threw: Aborted()" },
{ path: "other/thing.ts", error: "other/thing.ts: parse failed — crashed" },
];
writeGraph(g, outDir);
// These paths have no nodes at all (a real build gives a failed file no
// file node), so without the note they would read as "no definitions".
const failed = skeleton(dir, "broken.cs");
assert.match(failed.note ?? "", /failed to parse at build time/, JSON.stringify(failed));
assert.match(failed.note ?? "", /Aborted\(\)/, "the build error itself is shown");
assert.deepEqual(failed.entries, []);
// basename-suffix resolves to the failure too
const nested = skeleton(dir, "thing.ts");
assert.match(nested.note ?? "", /parse failed — crashed/);
// a file the build really parsed keeps its entries; the failed list must
// not swallow it
const ok = skeleton(dir, "api.ts");
assert.equal(ok.entries.length, 1, "a parsed file is untouched by parseFailed");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

test("ask reports coverage: 1.0 when every query term hits, low on mostly-off-corpus prompts", async () => {
const dir = makeFixture();
try {
Expand Down
35 changes: 35 additions & 0 deletions test/generic-extract.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -823,3 +823,38 @@ test("a throwing grammar is a per-file build error, cached as a failure (#139)",
swapGrammarForTest("rust", prev);
}
});

// Issue #816: the graph meta carries the parse-failed files, so the search
// answers can blame the index instead of the pattern — on a fresh build and
// on the re-parse of a cached failure, and NOT on a clean build.
test("buildGraph meta carries parseFailed, fresh and re-parsed; a clean build has none (#816)", async () => {
const dir = mkdtempSync(join(tmpdir(), "graft-parse-failed-meta-"));
writeFileSync(join(dir, "lib.rs"), "pub fn f() {}\n");
await warmGenericGrammars(["rust"]);
const prev = swapGrammarForTest("rust", THROWING_GRAMMAR);
try {
await buildGraph(dir, { reuse: false });
const g = readGraph(wiringPath(contextDirFor(dir)));
assert.ok(g, "graph built");
assert.deepEqual(g!.meta.parseFailed?.map((f) => f.path), ["lib.rs"], "the failed file rides the meta");
assert.match(g!.meta.parseFailed?.[0]?.error ?? "", /rust grammar threw/, "with the build error");
await buildGraph(dir, { reuse: true });
const g2 = readGraph(wiringPath(contextDirFor(dir)));
assert.deepEqual(g2!.meta.parseFailed?.map((f) => f.path), ["lib.rs"], "the re-parsed failure still rides the meta");
} finally {
swapGrammarForTest("rust", prev);
rmSync(dir, { recursive: true, force: true });
}
// Clean build: the field is absent, and a search answer must not grow a
// parse-failure sentence for it.
const clean = mkdtempSync(join(tmpdir(), "graft-parse-clean-"));
try {
writeFileSync(join(clean, "ok.rs"), "pub fn g() {}\n");
await buildGraph(clean, { reuse: false });
const gc = readGraph(wiringPath(contextDirFor(clean)));
assert.ok(gc, "clean graph built");
assert.equal(gc!.meta.parseFailed, undefined, "clean build: no parseFailed");
} finally {
rmSync(clean, { recursive: true, force: true });
}
});
36 changes: 35 additions & 1 deletion test/search-grep.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { grepGraph } from '../src/search/grep.js';
import { zeroHitNote } from '../src/search/grep-cli.js';
import { zeroHitNote, formatGrepResult } from '../src/search/grep-cli.js';
import { WALK_RELATIONS } from '../src/graph/relations.js';
import { readGraph, wiringPath } from '../src/graph/write.js';
import type { GraphV1, NodeV1 } from '../src/graph/types.js';
Expand Down Expand Up @@ -308,3 +308,37 @@ test('zeroHitNote (grep-cli.ts): zero hits AND unreadable indexed files mentions
// ...but truncation is never silent: 1 unreadable file must be surfaced too.
assert.match(note, /1 indexed file.*could not be read/);
});

// Issue #816: parse failures ride the result and the answers blame the index,
// not the pattern. An old graph without meta.parseFailed answers exactly as
// before — the sentence must never appear from nothing.
test('grepResult carries parseFailed; the notes say what was never searched (#816)', () => {
const repo = needleRepo();
const graph = loadBuiltGraph(repo);
graph.meta.parseFailed = [
{ path: 'big/failed1.ts', error: 'big/failed1.ts: parse failed — c_sharp grammar threw: Aborted()' },
{ path: 'big/failed2.ts', error: 'big/failed2.ts: parse failed — c_sharp grammar threw: Aborted()' },
];
const r = grepGraph(graph, repo, 'NEEDLE');
assert.equal(r.parseFailed, 2, 'the count rides the result');

const hitText = formatGrepResult(r);
assert.match(hitText, /2 source files failed to parse and were not searched/);
assert.match(hitText, /use rg for exhaustive search/);

const zero = { ...r, totalHits: 0, groups: [] };
const note = zeroHitNote(zero);
assert.match(note, /no hits for "NEEDLE"/);
assert.match(note, /2 source files failed to parse and were not searched/);
assert.doesNotMatch(note, /All indexed code was searched/, 'that sentence is a lie when files failed');

// Adversarial: an old graph (no meta.parseFailed) must answer unchanged —
// no sentence from nothing, retry advice intact.
const old = loadBuiltGraph(repo);
delete old.meta.parseFailed;
const rOld = grepGraph(old, repo, 'NOPE_NOT_PRESENT');
assert.equal(rOld.parseFailed, 0);
const noteOld = zeroHitNote(rOld);
assert.doesNotMatch(noteOld, /failed to parse/);
assert.match(noteOld, /All indexed code was searched/);
});
Loading