diff --git a/docs/operations/monitoring.md b/docs/operations/monitoring.md index 3fb46f3704..e621434460 100644 --- a/docs/operations/monitoring.md +++ b/docs/operations/monitoring.md @@ -185,6 +185,8 @@ These are Sentry MCP application attributes that are not part of the MCP semanti - `app.server.mode.experimental` - Whether experimental tools are enabled for this MCP request or stdio session - `app.server.distribution` - stdio only: `@sentry/mcp-server` for the npm package, `sentry mcp` for the CLI command - `app.consent.skill..granted` - Per-skill boolean attributes for skills granted to the MCP request. Skill ids are normalized with `-` replaced by `_` +- `app.side_quest.` - `"shown"` or `"capped"` (detected but cut by the cap) for each side quest a tool call detected; unset when not detected. Ids are stable snake_case +- `app.side_quests.detected`, `app.side_quests.shown` - Integer counts of side quests detected and shown; `0` when detectors ran and found none. Each shown quest also increments the `app.side_quest.shown` counter metric, tagged `side_quest.id`, which trace sampling does not drop - `app.upstream.host` - Upstream Sentry host configured for the server - `app.url.full` - MCP URL configured for stdio diff --git a/packages/mcp-core/src/internal/issue-side-quests.test.ts b/packages/mcp-core/src/internal/issue-side-quests.test.ts index 82184cba02..ab0474a778 100644 --- a/packages/mcp-core/src/internal/issue-side-quests.test.ts +++ b/packages/mcp-core/src/internal/issue-side-quests.test.ts @@ -1,7 +1,10 @@ import { eventFixture, sourceMapDebugFixture } from "@sentry/mcp-server-mocks"; import { describe, expect, it } from "vitest"; import { EventSchema, SourceMapDebugSchema } from "../api-client/schema"; -import { buildIssueSideQuests } from "./issue-side-quests"; +import { + ISSUE_SIDE_QUEST_IDS, + buildIssueSideQuests, +} from "./issue-side-quests"; const session = { experimentalMode: false }; const files = ["/a.js", "/b.js", "/c.js"]; @@ -36,6 +39,14 @@ function eventWithFourFiles() { } describe("buildIssueSideQuests", () => { + it("uses unique snake_case ids, which analytics depend on", () => { + const ids = Object.values(ISSUE_SIDE_QUEST_IDS); + expect(new Set(ids).size).toBe(ids.length); + for (const id of ids) { + expect(id).toMatch(/^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$/); + } + }); + it("lists each file's cause when the causes differ", () => { const event = eventWithFourFiles(); const debug = SourceMapDebugSchema.parse(sourceMapDebugFixture); diff --git a/packages/mcp-core/src/internal/issue-side-quests.ts b/packages/mcp-core/src/internal/issue-side-quests.ts index 7421ec2435..d7dbf68de4 100644 --- a/packages/mcp-core/src/internal/issue-side-quests.ts +++ b/packages/mcp-core/src/internal/issue-side-quests.ts @@ -25,6 +25,16 @@ export interface IssueSideQuestInput { sourceMapDebug?: SourceMapDebug; } +/** + * The ids of the issue side quests. Analytics query them as span attributes + * and metric tags, so renaming one breaks dashboards. + */ +export const ISSUE_SIDE_QUEST_IDS = { + jsSourceMaps: "js_source_maps", + missingStackTrace: "missing_stack_trace", + missingUserIdentity: "missing_user_identity", +} as const; + const MAX_FILES = 3; const JS_SOURCE_MAPS_TROUBLESHOOTING_URL = @@ -180,7 +190,7 @@ function detectJsSourceMapFailure({ const fallback = `${matched.map(([, detail]) => detail.problem).join("; ")}. ${[...new Set(matched.map(([, detail]) => detail.fix))].join(" ")}`; const quest = (tip: string, details?: string[]): SideQuestCandidate => ({ rank: 0, - id: "js_source_maps", + id: ISSUE_SIDE_QUEST_IDS.jsSourceMaps, tip, docs: JS_SOURCE_MAPS_TROUBLESHOOTING_URL, details, @@ -314,7 +324,7 @@ function detectMissingUserIdentity({ return { rank: 10, - id: "missing_user_identity", + id: ISSUE_SIDE_QUEST_IDS.missingUserIdentity, tip: `${problem} Call ${call} after sign-in; an opaque ID is enough.`, docs, }; @@ -522,7 +532,7 @@ export function hasReadableStack(event: Event): boolean { } function missingStackTrace(tip: string, docs: string): SideQuestCandidate { - return { rank: 5, id: "missing_stack_trace", tip, docs }; + return { rank: 5, id: ISSUE_SIDE_QUEST_IDS.missingStackTrace, tip, docs }; } /** diff --git a/packages/mcp-core/src/internal/side-quests.test.ts b/packages/mcp-core/src/internal/side-quests.test.ts index 85b1cbbe3f..0412884ccf 100644 --- a/packages/mcp-core/src/internal/side-quests.test.ts +++ b/packages/mcp-core/src/internal/side-quests.test.ts @@ -1,10 +1,20 @@ -import { describe, expect, it } from "vitest"; +import { getActiveSpan, metrics } from "@sentry/core"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { type SideQuestCandidate, buildSideQuests, formatSideQuests, } from "./side-quests"; +vi.mock("@sentry/core", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getActiveSpan: vi.fn(), + metrics: { ...actual.metrics, count: vi.fn() }, + }; +}); + const session = { experimentalMode: false, availableToolNames: new Set(["get_issue_details", "search_events"]), @@ -126,6 +136,67 @@ describe("buildSideQuests", () => { }); }); +describe("side quest telemetry", () => { + const setAttributes = vi.fn(); + + beforeEach(() => { + setAttributes.mockClear(); + vi.mocked(metrics.count).mockClear(); + vi.mocked(getActiveSpan).mockReturnValue({ + setAttributes, + } as unknown as ReturnType); + }); + + it("marks each detected quest as shown or capped, with the counts", () => { + buildSideQuests( + null, + [ + detector({ rank: 10, id: "late", tip: "Late." }), + detector({ rank: 0, id: "first", tip: "First." }), + detector(null), + detector({ rank: 5, id: "second", tip: "Second." }), + ], + session, + 2, + ); + + expect(setAttributes).toHaveBeenCalledOnce(); + expect(setAttributes).toHaveBeenCalledWith({ + "app.side_quests.detected": 3, + "app.side_quests.shown": 2, + "app.side_quest.first": "shown", + "app.side_quest.second": "shown", + "app.side_quest.late": "capped", + }); + }); + + it("counts each shown quest, not the capped ones", () => { + buildSideQuests( + null, + [ + detector({ rank: 0, id: "first", tip: "First." }), + detector({ rank: 1, id: "capped", tip: "Capped." }), + ], + session, + 1, + ); + + expect(vi.mocked(metrics.count).mock.calls).toEqual([ + ["app.side_quest.shown", 1, { attributes: { "side_quest.id": "first" } }], + ]); + }); + + it("records zero counts when no detector finds anything", () => { + buildSideQuests(null, [detector(null)], session); + + expect(setAttributes).toHaveBeenCalledWith({ + "app.side_quests.detected": 0, + "app.side_quests.shown": 0, + }); + expect(metrics.count).not.toHaveBeenCalled(); + }); +}); + describe("formatSideQuests", () => { it("renders one line per quest, with its calls, docs, and details", () => { expect( diff --git a/packages/mcp-core/src/internal/side-quests.ts b/packages/mcp-core/src/internal/side-quests.ts index ffe448fd2f..4b598c2ffa 100644 --- a/packages/mcp-core/src/internal/side-quests.ts +++ b/packages/mcp-core/src/internal/side-quests.ts @@ -1,3 +1,4 @@ +import { getActiveSpan, metrics } from "@sentry/core"; import { inlineCode } from "@sentry/toolkit-core/markdown"; import { z } from "zod"; import { @@ -80,13 +81,18 @@ export function buildSideQuests( session: ToolSession, max = MAX_SIDE_QUESTS, ): SideQuest[] | undefined { - const quests = detectors + const ranked = detectors .flatMap((detect) => { const candidate = detect(input); return candidate ? [candidate] : []; }) // Array.prototype.sort is stable, so ties keep detector order - .sort((a, b) => a.rank - b.rank) + .sort((a, b) => a.rank - b.rank); + recordSideQuests( + ranked.map(({ id }) => id), + max, + ); + const quests = ranked .slice(0, max) .map(({ id, tip, next, docs, details }) => { const calls = (next ?? []) @@ -104,6 +110,29 @@ export function buildSideQuests( return quests.length > 0 ? quests : undefined; } +/** + * Records which quests were detected and which the cap cut, so analytics can + * query them without parsing tool output: one `app.side_quest.` span + * attribute per detected quest, the counts, and a metric per shown quest, + * which trace sampling does not drop. Quest ids are part of this contract. + */ +function recordSideQuests(ids: string[], max: number) { + const shown = ids.slice(0, max); + const attributes: Record = { + "app.side_quests.detected": ids.length, + "app.side_quests.shown": shown.length, + }; + ids.forEach((id, index) => { + attributes[`app.side_quest.${id}`] = index < max ? "shown" : "capped"; + }); + getActiveSpan()?.setAttributes(attributes); + for (const id of shown) { + metrics.count("app.side_quest.shown", 1, { + attributes: { "side_quest.id": id }, + }); + } +} + /** The `## Concerns Identified` markdown section, or an empty string. */ export function formatSideQuests(quests: SideQuest[] | undefined): string { if (!quests) { diff --git a/packages/mcp-core/src/server.test.ts b/packages/mcp-core/src/server.test.ts index ce97146c1c..c3f7c80345 100644 --- a/packages/mcp-core/src/server.test.ts +++ b/packages/mcp-core/src/server.test.ts @@ -53,6 +53,7 @@ vi.mock("@sentry/core", () => ({ captureException: vi.fn(), captureMessage: vi.fn(), getClient: vi.fn(), + metrics: { count: vi.fn() }, })); vi.mock("@sentry/core/server", () => ({