feat(groomer): recheck saved empty searches for freshness

prisma/migrations/20261003000000_add_issue_groomed_search_queries/migration.sql

@@ -0,0 +1,6 @@
+-- Preserve empty code-search queries so freshness checks can verify whether
+-- repo-wide negative evidence still holds after a default-branch commit.
+ALTER TABLE "Issue"
+  ADD COLUMN IF NOT EXISTS "groomedSearchCodeQueries" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[];
+
+COMMENT ON COLUMN "Issue"."groomedSearchCodeQueries" IS 'Successful empty search_code queries backing global grooming evidence';

prisma/schema.prisma

@@ -72,6 +72,7 @@ model Issue {
   groomedEvidenceCapturedAt DateTime?
   groomedEvidenceScope      String?
   groomedEvidencePaths      String[]  @default([])
+  groomedSearchCodeQueries  String[]  @default([])
   groomedDependencyKeys     String[]  @default([])
   groomedOpenBlockerKeys    String[]  @default([])
   groomedRelatedWork        Json?

src/lib/github-ci.ts

@@ -160,6 +160,27 @@ export async function fetchLatestCommit(repoFullName: string, branch: string): P
   return response.json();
 }
 
+/**
+ * Committer timestamp (ISO string) for an existing commit sha, or null when
+ * the ref does not resolve. Used by the grooming freshness pass to judge
+ * whether the code-search index has had time to catch up with a new head.
+ * GitHub always returns a committer date for a commit; the author date is
+ * deliberately not used as a fallback (it can make a rebased head look
+ * older than it is).
+ */
+export async function fetchCommitDate(repoFullName: string, ref: string): Promise<string | null> {
+  const response = await fetchWithRetry(`${GITHUB_API}/repos/${repoFullName}/commits/${encodeURIComponent(ref)}`, {
+    headers: await getHeadersAsync(),
+  });
+  if (!response.ok) {
+    if (response.status === 404) return null;
+    const text = await response.text();
+    throw new Error(`Failed to fetch commit ${ref} for ${repoFullName}: ${response.status} ${text}`);
+  }
+  const data = (await response.json()) as { commit?: { committer?: { date?: string } } };
+  return data.commit?.committer?.date ?? null;
+}
+
 export function jobIdFromCheckRunUrl(url: string | undefined | null): string | null {
   const m = /\/job\/(\d+)/.exec(url ?? "");
   return m ? m[1] : null;

src/lib/github-code-search.test.ts

@@ -137,6 +137,14 @@ describe("github-code-search: searchRepositoryCode (the groomer repo-exploration
     expect(fetchSpy).toHaveBeenCalledTimes(1);
   });
 
+  it("fails an incomplete search with no items instead of reporting no matches", async () => {
+    fetchSpy.mockResolvedValueOnce(mockResponse({ total_count: 0, incomplete_results: true, items: [] }));
+
+    await expect(searchRepositoryCode("org/repo", "prisma", 5)).rejects.toThrow(
+      /^Code search failed for org\/repo: search timed out with incomplete results and no matches$/,
+    );
+  });
+
   // (c) HTTP 429 retried with backoff.
   it("retries a 429 rate-limit response with backoff and succeeds on the next attempt", async () => {
     fetchSpy
@@ -306,10 +314,17 @@ describe("github-code-search: compareCommits (grooming freshness, #1064)", () =>
       mockResponse({
         status: "ahead",
         files: [{ filename: "src/new.ts", previous_filename: "src/old.ts" }, { filename: "docs/a.md" }],
+        commits: [{ commit: { committer: { date: "2026-09-28T00:00:00Z" } } }],
       }),
     );
     const result = await compareCommits("org/repo", "base1", "head1");
-    expect(result).toEqual({ ok: true, status: "ahead", files: ["src/new.ts", "src/old.ts", "docs/a.md"], truncated: false });
+    expect(result).toEqual({
+      ok: true,
+      status: "ahead",
+      files: ["src/new.ts", "src/old.ts", "docs/a.md"],
+      truncated: false,
+      firstCommitDate: "2026-09-28T00:00:00Z",
+    });
     expect(fetchSpy).toHaveBeenCalledTimes(1);
     expect(String(fetchSpy.mock.calls[0][0])).toContain("/repos/org/repo/compare/base1...head1?per_page=1");
   });

src/lib/github-code-search.ts

@@ -57,11 +57,19 @@ export async function searchRepositoryCode(
     // so a search with more matches than fit on one page is not silently
     // truncated at the first page. Each page is fetched through fetchWithRetry,
     // so transient 429/5xx responses are retried here as well.
-    const items = await fetchPaginated<{ path?: string; html_url?: string }>(
-      url,
-      limit,
-      (data) => (data as { items?: { path?: string; html_url?: string }[] }).items ?? [],
-    );
+    let incomplete = false;
+    const items = await fetchPaginated<{ path?: string; html_url?: string }>(url, limit, (data) => {
+      const page = data as { items?: { path?: string; html_url?: string }[]; incomplete_results?: boolean };
+      if (page.incomplete_results === true) incomplete = true;
+      return page.items ?? [];
+    });
+    // Code search can time out and still answer 200 with incomplete_results
+    // and no items. That is not "no matches": the groomer treats an empty
+    // result as proof of absence and the freshness recheck advances on it
+    // (#1091), so surface it as a failure instead.
+    if (incomplete && items.length === 0) {
+      throw new Error("search timed out with incomplete results and no matches");
+    }
     return items.map((item) => ({
       path: item.path ?? "",
       url: item.html_url ?? "",
@@ -161,6 +169,12 @@ export type CommitComparison =
       files: string[];
       /** True when GitHub's file cap was reached, so `files` may be incomplete. */
       truncated: boolean;
+      /**
+       * Committer timestamp of the first (oldest) commit in base...head, when
+       * the response carried one. Lets callers bound how long a recheck may
+       * stay deferred on an unverified range (#1091).
+       */
+      firstCommitDate?: string | null;
     }
   | {
       ok: false;
@@ -196,18 +210,22 @@ export async function compareCommits(
     const data = (await response.json()) as {
       status?: string;
       files?: Array<{ filename?: string; previous_filename?: string }>;
+      commits?: Array<{ commit?: { committer?: { date?: string } } }>;
     };
     const rawFiles = Array.isArray(data.files) ? data.files : [];
     const files = new Set<string>();
     for (const file of rawFiles) {
       if (typeof file.filename === "string" && file.filename) files.add(file.filename);
       if (typeof file.previous_filename === "string" && file.previous_filename) files.add(file.previous_filename);
     }
+    // per_page=1 still returns the first (oldest) commit of the range.
+    const firstCommitDate = Array.isArray(data.commits) ? data.commits[0]?.commit?.committer?.date ?? null : null;
     return {
       ok: true,
       status: typeof data.status === "string" ? data.status : "unknown",
       files: [...files],
       truncated: rawFiles.length >= COMPARE_MAX_FILES,
+      firstCommitDate,
     };
   } catch (err) {
     return {

src/lib/github-facades.test.ts

@@ -34,6 +34,7 @@ describe("github domain modules expose expected exports", () => {
   it("github-ci exports CI/workflows/runs/jobs/releases/packages/commits/logs symbols", () => {
     expect(Object.keys(Ci).sort()).toEqual([
       "extractLogExcerpt",
+      "fetchCommitDate",
       "fetchFailedJobLogExcerpt",
       "fetchLatestCommit",
       "fetchPackages",

src/lib/groomer/freshness-invalidation.test.ts

@@ -34,6 +34,7 @@ function row(overrides: Partial<FreshnessIssueRow> = {}): FreshnessIssueRow {
     groomedEvidenceCapturedAt: CAPTURED,
     groomedEvidenceScope: "paths",
     groomedEvidencePaths: ["src/a.ts"],
+    groomedSearchCodeQueries: [],
     groomedDependencyKeys: [],
     groomedOpenBlockerKeys: [],
     groomedRelatedWork: null,
@@ -86,9 +87,12 @@ function fakeStore(rows: FreshnessIssueRow[]): FakeStore {
 function fakeGitHub() {
   return {
     fetchHeadSha: vi.fn(async (): Promise<string | null> => "sha-1"),
+    searchCode: vi.fn(async (): Promise<{ path: string }[]> => []),
     compareCommits: vi.fn(
       async (): Promise<CommitComparison> => ({ ok: true, status: "ahead", files: [], truncated: false }),
     ),
+    // Old enough that the code-search index is assumed caught up (#1091).
+    fetchCommitDate: vi.fn(async (): Promise<string | null> => new Date(Date.now() - 3_600_000).toISOString()),
     fetchRecentComments: vi.fn(async (): Promise<Array<{ author: string; createdAt: string }>> => []),
     fetchIssueState: vi.fn(async (): Promise<"open" | "closed" | null> => "open"),
     fetchPullRequestState: vi.fn(async (): Promise<"open" | "closed" | "merged" | null> => "open"),
@@ -263,12 +267,190 @@ describe("runGroomingFreshnessPass", () => {
       expect(store.stale.get("issue-1")?.detail).toContain("src/a.ts");
     });
 
-    it("conservatively stales global (negative) evidence on any commit", async () => {
+    it("conservatively stales global evidence without saved search queries", async () => {
       github.fetchHeadSha.mockResolvedValue("sha-2");
       github.compareCommits.mockResolvedValue({ ok: true, status: "ahead", files: ["docs/readme.md"], truncated: false });
       const store = fakeStore([{ ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [] }]);
       await pass(store, github);
       expect(store.stale.get("issue-1")?.reasons).toEqual(["global_evidence_commit"]);
+      expect(github.searchCode).not.toHaveBeenCalled();
+    });
+
+    it("advances global evidence when saved empty queries remain empty", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({ ok: true, status: "ahead", files: ["docs/readme.md"], truncated: false });
+      const store = fakeStore([{ ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: ["new symbol", "missing call"] }]);
+      await pass(store, github);
+      expect(store.stale.size).toBe(0);
+      expect(store.advanced).toEqual([{ id: "issue-1", data: { groomingVerifiedSha: "sha-2" } }]);
+      expect(github.searchCode).toHaveBeenCalledTimes(2);
+    });
+
+    it("stales global evidence when a saved query now matches", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({ ok: true, status: "ahead", files: [], truncated: false });
+      github.searchCode.mockResolvedValueOnce([]).mockResolvedValueOnce([{ path: "src/new.ts" }]);
+      const store = fakeStore([{ ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: ["first", "specific missing query"] }]);
+      await pass(store, github);
+      expect(store.stale.get("issue-1")?.reasons).toEqual(["global_evidence_commit"]);
+      expect(store.stale.get("issue-1")?.detail).toContain("specific missing query");
+    });
+
+    it("defers instead of staling when the search budget runs out mid-recheck", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({
+        ok: true,
+        status: "ahead",
+        files: [],
+        truncated: false,
+        firstCommitDate: new Date(Date.now() - 5 * 60_000).toISOString(),
+      });
+      const store = fakeStore([{ ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: ["first", "second"] }]);
+      await pass(store, github, { ...DEFAULT_FRESHNESS_BUDGET, maxSearchCodeRechecks: 2 });
+      // One budget unit goes to the commit-date check, one to the first query;
+      // the second query exhausts the budget, which defers rather than stales.
+      expect(store.stale.size).toBe(0);
+      expect(store.advanced).toEqual([]);
+      expect(github.searchCode).toHaveBeenCalledTimes(1);
+    });
+
+    it("stales a persistently exhausted issue once the oldest unverified commit passes the defer limit", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({
+        ok: true,
+        status: "ahead",
+        files: [],
+        truncated: false,
+        firstCommitDate: new Date(Date.now() - 3 * 3_600_000).toISOString(),
+      });
+      const store = fakeStore([{ ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: ["first", "second"] }]);
+      await pass(store, github, { ...DEFAULT_FRESHNESS_BUDGET, maxSearchCodeRechecks: 2 });
+      expect(store.stale.get("issue-1")?.reasons).toEqual(["global_evidence_commit"]);
+      expect(store.advanced).toEqual([]);
+    });
+
+    it("defers within the bound when the budget is exhausted before the date fetch", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({
+        ok: true,
+        status: "ahead",
+        files: [],
+        truncated: false,
+        firstCommitDate: new Date(Date.now() - 5 * 60_000).toISOString(),
+      });
+      const store = fakeStore([{ ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: ["query"] }]);
+      const result = await pass(store, github, { ...DEFAULT_FRESHNESS_BUDGET, maxSearchCodeRechecks: 0 });
+      expect(store.stale.size).toBe(0);
+      expect(store.advanced).toEqual([]);
+      expect(github.searchCode).not.toHaveBeenCalled();
+      expect(result.deferred).toBe(1);
+    });
+
+    it("completes the recheck for an issue with the maximum saved queries", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({
+        ok: true,
+        status: "ahead",
+        files: [],
+        truncated: false,
+        firstCommitDate: new Date(Date.now() - 5 * 60_000).toISOString(),
+      });
+      const queries = Array.from({ length: 10 }, (_, index) => `missing ${index}`);
+      const store = fakeStore([{ ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: queries }]);
+      await pass(store, github);
+      // Commit-date fetch (1) plus all ten queries fit the default budget.
+      expect(store.stale.size).toBe(0);
+      expect(store.advanced).toEqual([{ id: "issue-1", data: { groomingVerifiedSha: "sha-2" } }]);
+      expect(github.searchCode).toHaveBeenCalledTimes(10);
+      expect(github.fetchCommitDate).toHaveBeenCalledTimes(1);
+    });
+
+    it("defers the recheck while the new head is younger than the index grace window", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({
+        ok: true,
+        status: "ahead",
+        files: [],
+        truncated: false,
+        firstCommitDate: new Date(Date.now() - 5 * 60_000).toISOString(),
+      });
+      github.fetchCommitDate.mockResolvedValue(new Date(Date.now() - 60_000).toISOString());
+      const store = fakeStore([{ ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: ["query"] }]);
+      const result = await pass(store, github);
+      expect(store.stale.size).toBe(0);
+      expect(store.advanced).toEqual([]);
+      expect(github.searchCode).not.toHaveBeenCalled();
+      expect(result.deferred).toBe(1);
+    });
+
+    it("bounds the deferral: stales once the oldest unverified commit outlives the defer limit", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({
+        ok: true,
+        status: "ahead",
+        files: [],
+        truncated: false,
+        firstCommitDate: new Date(Date.now() - 3 * 3_600_000).toISOString(),
+      });
+      github.fetchCommitDate.mockResolvedValue(new Date(Date.now() - 60_000).toISOString());
+      const store = fakeStore([{ ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: ["query"] }]);
+      const result = await pass(store, github);
+      expect(store.stale.get("issue-1")?.reasons).toEqual(["global_evidence_commit"]);
+      expect(store.advanced).toEqual([]);
+      expect(result.deferred).toBe(0);
+    });
+
+    it("stales instead of deferring when the oldest unverified commit age is unknown", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({ ok: true, status: "ahead", files: [], truncated: false });
+      github.fetchCommitDate.mockResolvedValue(new Date(Date.now() - 60_000).toISOString());
+      const store = fakeStore([{ ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: ["query"] }]);
+      await pass(store, github);
+      expect(store.stale.get("issue-1")?.reasons).toEqual(["global_evidence_commit"]);
+    });
+
+    it("falls back to global staleness when the commit date is missing or unavailable", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({ ok: true, status: "ahead", files: [], truncated: false });
+      const issue = { ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: ["query"] };
+      github.fetchCommitDate.mockResolvedValue(null);
+      const missing = fakeStore([issue]);
+      await pass(missing, github);
+      expect(missing.stale.get("issue-1")?.reasons).toEqual(["global_evidence_commit"]);
+
+      github.fetchCommitDate.mockRejectedValueOnce(new Error("boom"));
+      const failed = fakeStore([issue]);
+      await pass(failed, github);
+      expect(failed.stale.get("issue-1")?.reasons).toEqual(["global_evidence_commit"]);
+    });
+
+    it("stales a global result whose relied-on read path the commit touched, without rechecking", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({ ok: true, status: "ahead", files: ["src/a.ts"], truncated: false });
+      const store = fakeStore([
+        { ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: ["src/a.ts"], groomedSearchCodeQueries: ["query"] },
+      ]);
+      await pass(store, github);
+      expect(store.stale.get("issue-1")?.reasons).toEqual(["global_evidence_commit"]);
+      expect(store.stale.get("issue-1")?.detail).toContain("src/a.ts");
+      expect(github.searchCode).not.toHaveBeenCalled();
+    });
+
+    it("falls back to global staleness when search is missing or fails", async () => {
+      github.fetchHeadSha.mockResolvedValue("sha-2");
+      github.compareCommits.mockResolvedValue({ ok: true, status: "ahead", files: [], truncated: false });
+      const issue = { ...row(), groomedEvidenceScope: "global", groomedEvidencePaths: [], groomedSearchCodeQueries: ["query"] };
+      const missing = fakeStore([issue]);
+      const withoutSearch = { ...github } as { searchCode?: typeof github.searchCode } & Omit<ReturnType<typeof fakeGitHub>, "searchCode">;
+      delete withoutSearch.searchCode;
+      await pass(missing, withoutSearch as ReturnType<typeof fakeGitHub>);
+      expect(missing.stale.get("issue-1")?.reasons).toEqual(["global_evidence_commit"]);
+
+      github.searchCode.mockRejectedValueOnce(new Error("rate limited"));
+      const failed = fakeStore([issue]);
+      await pass(failed, github);
+      expect(failed.stale.get("issue-1")?.reasons).toEqual(["global_evidence_commit"]);
+      expect(failed.stale.get("issue-1")?.detail).toBe("default branch moved sha-1...sha-2 and the result relied on repo-wide evidence");
     });
 
     it("ignores commits for a result that used no repository evidence", async () => {