Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/ask/ask.ts
Original file line number Diff line number Diff line change
Expand Up @@ -201,10 +201,10 @@ function loadCorpus(outDir: string): Corpus {
* and land as the top hit (observed: an `ask` returning a `Test…` function first,
* sending the agent to the wrong file). A multiplicative de-rank keeps tests in
* the results (they still matter for "where are the tests") but below the real
* definition. Covers Go (`_test.go`), JS/TS (`.test.` / `.spec.`), and test dirs. */
export function isTestPath(path: string): boolean {
return /(^|\/)(tests?|__tests__|spec)\/|(_test|\.test|\.spec)\.[a-z]+$|(^|\/)(test_[^/]+|conftest)\.py$/i.test(path || "");
}
* definition. The predicate lives in the search layer (grep.ts) so grep and
* callers share it (#425). */
import { isTestPath } from "../search/grep.js";
export { isTestPath };
const TEST_RANK_PENALTY = 0.35;

function score(
Expand Down
10 changes: 8 additions & 2 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1111,13 +1111,14 @@ grouped(program.command("callers"), GROUP.find)
.option("--direction <in|out>", 'edge direction: "in" = callers (default), "out" = callees')
.option("-d, --depth <n>", 'walk transitively up to N hops for blast radius, or "all" for the full connected closure (default 1)')
.option("--in <path>", "narrow matches to nodes at or under this path prefix")
.option("--no-tests", "leave test-file hits out (paths like *.test.ts, _test.go, conftest.py) — default labels them [test] and lists them last")
.option("--json", "output as JSON")
.option(...NO_REFRESH_FLAG)
.action(
async (
symbol: string,
dirArg: string | undefined,
opts: { direction?: string; depth?: string; in?: string; json?: boolean; refresh?: boolean },
opts: { direction?: string; depth?: string; in?: string; json?: boolean; tests?: boolean; refresh?: boolean },
) => {
const dir = noteQuery(queryRoot(dirArg));
await refreshBefore(dir, opts);
Expand All @@ -1129,6 +1130,7 @@ grouped(program.command("callers"), GROUP.find)
? (/^(all|full|max)$/i.test(opts.depth) ? Number.POSITIVE_INFINITY : Number(opts.depth))
: undefined,
in: opts.in,
includeTests: opts.tests !== false, // #425
});
return;
}
Expand All @@ -1138,6 +1140,7 @@ grouped(program.command("callers"), GROUP.find)
depth: opts.depth,
in: opts.in,
json: opts.json,
includeTests: opts.tests !== false, // #425
globalDir: globalOpts.dir,
});
},
Expand Down Expand Up @@ -1182,26 +1185,29 @@ grouped(program.command("grep"), GROUP.find)
.option("-i, --ignore-case", "case-insensitive match")
.option("--fixed", "treat pattern as a literal string, not a regex")
.option("--in <path>", "narrow to files at or under this path prefix")
.option("--no-tests", "leave test files out (paths like *.test.ts, _test.go, conftest.py) — default labels them [test] and lists them last")
.option("--json", "output as JSON")
.option(...NO_REFRESH_FLAG)
.action(
async (
pattern: string,
dirArg: string | undefined,
opts: { ignoreCase?: boolean; fixed?: boolean; in?: string; json?: boolean; refresh?: boolean },
opts: { ignoreCase?: boolean; fixed?: boolean; in?: string; json?: boolean; tests?: boolean; refresh?: boolean },
) => {
const dir = noteQuery(queryRoot(dirArg));
await refreshBefore(dir, opts);
const globalOpts = program.opts<{ dir?: string }>();
if (readWorkspace(dir, globalOpts.dir)) {
runWorkspaceGrep(dir, globalOpts.dir, pattern, {
ignoreCase: opts.ignoreCase, fixed: opts.fixed, json: opts.json,
includeTests: opts.tests !== false, // #425
});
return;
}
const { runGrepCommand } = await import("./search/grep-cli.js");
runGrepCommand(pattern, dir, {
ignoreCase: opts.ignoreCase,
includeTests: opts.tests !== false, // #425
fixed: opts.fixed,
in: opts.in,
json: opts.json,
Expand Down
22 changes: 20 additions & 2 deletions src/graph/traverse-cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import { contextDirFor } from "../context/node-file.js";
import { withSavings, savingsFor, type Savings } from "../context/savings.js";
import { loadGraphCached } from "./load.js";
import { resolveSymbol, edgeWalk, type Direction, type EdgeHit } from "./traverse.js";
import { isTestPath } from "../search/grep.js";
import type { GraphV1, NodeV1 } from "./types.js";
import { cmd } from "../brand.js";

Expand All @@ -29,6 +30,8 @@ export interface CallersCliOptions {
depth?: string;
/** the top-level `--dir` override, so this command respects it like every other. */
globalDir?: string;
/** #425: false = leave test-file edges out (--no-tests). Default true (labelled). */
includeTests?: boolean;
}

const ARROW: Record<Direction, "←" | "→"> = { in: "←", out: "→" };
Expand All @@ -51,11 +54,25 @@ export function headerOf(n: NodeV1): string {
export function hitLine(direction: Direction, hit: EdgeHit, showDepth: boolean, quote?: Quote): string {
const arrow = ARROW[direction];
const depthTag = showDepth ? ` [depth ${hit.depth}]` : "";
const label = hit.node ? `${hit.node.name} (${hit.node.path}:${hit.node.span})` : `${hit.id} (unresolved import)`;
// #425: a caller (or callee) in a test file says so — it must not read as a
// production dependency.
const testTag = hit.node && isTestPath(hit.node.path) ? " [test]" : "";
const label = hit.node ? `${hit.node.name} (${hit.node.path}:${hit.node.span})${testTag}` : `${hit.id} (unresolved import)`;
const line = ` ${hit.relation} ${arrow} ${label}${depthTag}`;
return quote ? `${line}\n ${quote.n}: ${quote.text.trim()}` : line;
}

/**
* #425: with `includeTests: false`, edges touching a test file are dropped;
* otherwise (default) they are kept and the list is ordered production-first —
* a stable partition, so the BFS order inside each half is untouched. Hits
* with no resolved node have no path to judge and stay.
*/
export function orderAndFilterTestHits(hits: EdgeHit[], includeTests: boolean): EdgeHit[] {
const kept = includeTests ? hits : hits.filter((h) => !h.node || !isTestPath(h.node.path));
return [...kept.filter((h) => !h.node || !isTestPath(h.node.path)), ...kept.filter((h) => h.node && isTestPath(h.node.path))];
}

/** A quoted source line: where the edge actually happens. */
interface Quote {
n: number;
Expand Down Expand Up @@ -190,6 +207,7 @@ export function runCallersCommand(query: string, dir: string, opts: CallersCliOp
}

const direction = resolveDirection(opts.direction);
const includeTests = opts.includeTests !== false; // #425: tests included (labelled) unless --no-tests
let depth = DEFAULT_DEPTH;
if (opts.depth !== undefined) {
// `all` (aka full/max) = the whole transitive closure: walk until no new
Expand All @@ -210,7 +228,7 @@ export function runCallersCommand(query: string, dir: string, opts: CallersCliOp
}
const showDepth = depth > 1;

const results = matches.map((symbol) => ({ symbol, hits: edgeWalk(graph, symbol, direction, depth) }));
const results = matches.map((symbol) => ({ symbol, hits: orderAndFilterTestHits(edgeWalk(graph, symbol, direction, depth), includeTests) }));
const saved = callersSavings(graph, results);

if (opts.json) {
Expand Down
6 changes: 3 additions & 3 deletions src/graph/workspace-cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,9 +156,9 @@ export function runWorkspaceGrep(
root: string,
override: string | undefined,
pattern: string,
opts: { ignoreCase?: boolean; fixed?: boolean; json?: boolean },
opts: { ignoreCase?: boolean; fixed?: boolean; json?: boolean; includeTests?: boolean },
): void {
const { result, coverage } = federateGrep(root, override, pattern, { ignoreCase: opts.ignoreCase, fixed: opts.fixed });
const { result, coverage } = federateGrep(root, override, pattern, { ignoreCase: opts.ignoreCase, fixed: opts.fixed, includeTests: opts.includeTests });
if (opts.json) {
console.log(JSON.stringify(result, null, 2));
return;
Expand Down Expand Up @@ -189,7 +189,7 @@ export function runWorkspaceCallers(
root: string,
override: string | undefined,
symbol: string,
opts: { direction?: Direction; depth?: number; in?: string },
opts: { direction?: Direction; depth?: number; in?: string; includeTests?: boolean },
): void {
const { text, found } = federateCallers(root, override, symbol, opts);
if (!found) {
Expand Down
8 changes: 5 additions & 3 deletions src/graph/workspace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import {
looseNoteFor,
} from "./traverse-cli.js";
import { edgeWalk, resolveSymbol, type Direction } from "./traverse.js";
import { orderAndFilterTestHits } from "./traverse-cli.js";
import { wiringPath } from "./write.js";
import type { GraphV1 } from "./types.js";
import {
Expand Down Expand Up @@ -565,7 +566,7 @@ export function federateGrep(
root: string,
override: string | undefined,
pattern: string,
opts: { ignoreCase?: boolean; fixed?: boolean } = {},
opts: { ignoreCase?: boolean; fixed?: boolean; includeTests?: boolean } = {},
): { result: GrepResult; coverage: string } {
const wg = loadWorkspaceGraphs(root, override);
const groups: GrepGroup[] = [];
Expand All @@ -579,6 +580,7 @@ export function federateGrep(
const r = grepGraph(graph, join(root, child), pattern, {
ignoreCase: opts.ignoreCase,
fixed: opts.fixed,
includeTests: opts.includeTests,
});
filesSearched += r.filesSearched;
totalHits += r.totalHits;
Expand Down Expand Up @@ -661,7 +663,7 @@ export function federateCallers(
root: string,
override: string | undefined,
symbol: string,
opts: { direction?: Direction; depth?: number; in?: string } = {},
opts: { direction?: Direction; depth?: number; in?: string; includeTests?: boolean } = {},
): { text: string; found: boolean } {
const wg = loadWorkspaceGraphs(root, override);
const direction: Direction = opts.direction ?? "in";
Expand All @@ -674,7 +676,7 @@ export function federateCallers(
const matches = resolveSymbol(graph, symbol, opts.in ? { in: opts.in } : {});
if (matches.length === 0) continue;
found = true;
const results = matches.map((m) => ({ symbol: m, hits: edgeWalk(graph, m, direction, depth) }));
const results = matches.map((m) => ({ symbol: m, hits: orderAndFilterTestHits(edgeWalk(graph, m, direction, depth), opts.includeTests !== false) }));
const lines = [`## ${child}/`];
for (const { symbol: sym, hits } of results) {
lines.push(headerOf(sym));
Expand Down
16 changes: 11 additions & 5 deletions src/search/grep-cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { resolve } from "node:path";
import { contextDirFor } from "../context/node-file.js";
import { withSavings } from "../context/savings.js";
import { loadGraphCached } from "../graph/load.js";
import { grepGraph, type GrepGroup, type GrepResult } from "./grep.js";
import { grepGraph, isTestPath, type GrepGroup, type GrepResult } from "./grep.js";
import { cmd } from "../brand.js";
import { ignoredCode, ignoredLabel } from "./ignored-code.js";

Expand All @@ -20,13 +20,16 @@ export interface GrepCliOptions {
fixed?: boolean;
in?: string;
json?: boolean;
/** #425: false = leave test files out entirely (--no-tests). Default true. */
includeTests?: boolean;
/** the top-level `--dir` override, so this command respects it like every other. */
globalDir?: string;
}

function groupHeader(g: GrepGroup): string {
if (g.symbol) return `${g.symbol.name} · ${g.symbol.kind} · ${g.symbol.path}:${g.symbol.span} · ${g.inDegree} in-edges`;
return `${g.path} (module level) · ${g.inDegree} in-edges`;
const testTag = isTestPath(g.path) ? " [test]" : ""; // #425: name the test file, don't let it read as production
if (g.symbol) return `${g.symbol.name} · ${g.symbol.kind} · ${g.symbol.path}:${g.symbol.span}${testTag} · ${g.inDegree} in-edges`;
return `${g.path}${testTag} (module level) · ${g.inDegree} in-edges`;
}

function formatGroup(g: GrepGroup): string {
Expand Down Expand Up @@ -59,7 +62,10 @@ export function formatGrepResult(result: GrepResult): string {
// line is a header — see withSavings).
const note = truncationNote(result);
const head = note ? `${formatGrepHeader(result)}\n${note}` : formatGrepHeader(result);
const blocks = [head, "", ...result.groups.map((g) => formatGroup(g) + "\n")];
// #425: production groups first, test groups after — a stable partition, so
// the in-degree ordering inside each half is untouched.
const ordered = [...result.groups.filter((g) => !isTestPath(g.path)), ...result.groups.filter((g) => isTestPath(g.path))];
const blocks = [head, "", ...ordered.map((g) => formatGroup(g) + "\n")];
const out = blocks.join("\n").replace(/\n+$/, "\n");
return withSavings(out, result.saved);
}
Expand Down Expand Up @@ -103,7 +109,7 @@ export function runGrepCommand(pattern: string, dir: string, opts: GrepCliOption

let result: GrepResult;
try {
result = grepGraph(graph, root, pattern, { ignoreCase: opts.ignoreCase, fixed: opts.fixed, in: opts.in });
result = grepGraph(graph, root, pattern, { ignoreCase: opts.ignoreCase, fixed: opts.fixed, in: opts.in, includeTests: opts.includeTests });
} catch (err) {
// `new RegExp` throws SyntaxError; anything else (an unindexed `--in` prefix)
// is a different mistake and must not be reported as a bad pattern.
Expand Down
27 changes: 26 additions & 1 deletion src/search/grep.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,35 @@ export interface GrepOptions {
/** Stop collecting hits after this many; the rest are tallied into
* `truncated.hits`. Default 300. */
maxHits?: number;
/** Leave test files out entirely (#425) — paths matching isTestPath.
* Default true: tests are included, labelled by the formatters. Excluded
* files are not counted into `filesSearched` (they were not searched). */
includeTests?: boolean;
}

const DEFAULT_MAX_HITS = 300;
const MAX_HIT_TEXT = 160;
const SPAN_RE = /^L(\d+)-L(\d+)$/;

/** Matches a test file's repo-relative path: a test directory (`tests/`,
* `test/`, `__tests__/`, `spec/`), a test-suffixed file (`.test.ts`, `.spec.js`,
* `_test.go`), or a Python test module (`test_*.py`, `conftest.py`). The
* `.test.` / `.spec.` forms need the surrounding dots so `testing.ts` and
* `latest.ts` stay production. Case-insensitive: Windows-checked trees use
* `Service.Test.ts` in the wild. */
const TEST_PATH_RE = /(^|\/)(tests?|__tests__|spec)\/|(_test|\.test|\.spec)\.[a-z]+$|(^|\/)(test_[^/]+|conftest)\.py$/i;

/**
* Whether `path` looks like a test file. Shared by every find surface that
* needs test awareness (#425): `ask` de-ranks them (TEST_RANK_PENALTY),
* `grep`/`callers` label and can exclude them (`--no-tests`), `blast` already
* separated `testModules`. Lives in the search layer so the graph layer and
* the CLI formatters can use it without importing the ask layer.
*/
export function isTestPath(path: string): boolean {
return TEST_PATH_RE.test(path || "");
}

function escapeRegExp(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
Expand Down Expand Up @@ -154,7 +177,9 @@ export function grepGraph(graph: GraphV1, repoRoot: string, pattern: string, opt
const inPrefix = opts.in ? normalizePathPrefix(opts.in) : undefined;
if (inPrefix) assertPrefixIndexed(graph, inPrefix);
const fileNodes = graph.nodes.filter(
(n) => n.kind === "file" && (inPrefix === undefined || pathUnderPrefix(n.path, inPrefix)),
(n) => n.kind === "file"
&& (inPrefix === undefined || pathUnderPrefix(n.path, inPrefix))
&& (opts.includeTests !== false || !isTestPath(n.path)), // #425: --no-tests
);
const symbolsByFile = symbolsByPath(graph);

Expand Down
52 changes: 50 additions & 2 deletions test/search-grep.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@
*/
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { grepGraph } from '../src/search/grep.js';
import { zeroHitNote } from '../src/search/grep-cli.js';
import { zeroHitNote, formatGrepResult } from '../src/search/grep-cli.js';
import { isTestPath } from '../src/search/grep.js';
import type { GrepResult } from '../src/search/grep.js';
import { WALK_RELATIONS } from '../src/graph/relations.js';
import { readGraph, wiringPath } from '../src/graph/write.js';
import type { GraphV1, NodeV1 } from '../src/graph/types.js';
Expand Down Expand Up @@ -308,3 +310,49 @@ test('zeroHitNote (grep-cli.ts): zero hits AND unreadable indexed files mentions
// ...but truncation is never silent: 1 unreadable file must be surfaced too.
assert.match(note, /1 indexed file.*could not be read/);
});

/*
* Issue #425: test files flowed into `grep` unlabelled and unfilterable — in
* the reporter's production repro ~80% of a grep answer was test lines. The
* ask layer has had the detection and a rank penalty since #37; these tests
* pin the search layer getting the same wiring: an `includeTests: false`
* filter on grepGraph (excluded files are not counted as searched), and a
* `[test]` label + production-first ordering in the text formatter.
*/
test('grepGraph with includeTests: false drops test files from groups and from filesSearched (#425)', () => {
const d = mkdtempSync(join(tmpdir(), 'graft-grep-notests-'));
try {
mkdirSync(join(d, 'src'), { recursive: true });
writeFileSync(join(d, 'src', 'service.ts'), 'export function formatName(n: string): string { return NEEDLE(n); }\n');
writeFileSync(join(d, 'src', 'service.test.ts'), 'it("trims", () => expect(NEEDLE(" a ")).toBe("a"));\n');
const graph = graphOf([fileNode('src/service.ts', 2), fileNode('src/service.test.ts', 2)]);

const withTests = grepGraph(graph, d, 'NEEDLE');
assert.deepEqual(withTests.groups.map((g) => g.path).sort(), ['src/service.test.ts', 'src/service.ts'], 'default keeps both files');
assert.equal(withTests.filesSearched, 2);

const noTests = grepGraph(graph, d, 'NEEDLE', { includeTests: false });
assert.deepEqual(noTests.groups.map((g) => g.path), ['src/service.ts'], 'only production remains');
assert.equal(noTests.filesSearched, 1, 'the excluded test file was not searched, so it is not counted');
assert.equal(noTests.totalHits, 1);
} finally {
rmSync(d, { recursive: true, force: true });
}
});

test('formatGrepResult labels test-file groups [test] and lists them after production (#425)', () => {
const result: GrepResult = {
pattern: 'NEEDLE',
filesSearched: 2,
totalHits: 2,
groups: [
{ symbol: null, path: 'src/service.test.ts', inDegree: 0, hits: [{ line: 1, text: 'NEEDLE inside the test' }] },
{ symbol: null, path: 'src/service.ts', inDegree: 0, hits: [{ line: 1, text: 'NEEDLE in production' }] },
],
truncated: { files: 0, hits: 0 },
};
const out = formatGrepResult(result);
assert.ok(/\[test\]/.test(out), 'the test group carries a [test] label');
assert.ok(!/service\.ts.*\[test\]/.test(out), 'the production file is not labelled');
assert.ok(out.indexOf('NEEDLE in production') < out.indexOf('NEEDLE inside the test'), 'the test group is listed after production, even though it sorted first');
});
Loading
Loading