Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/operations/monitoring.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,8 @@ These are Sentry MCP application attributes that are not part of the MCP semanti
- `app.server.mode.experimental` - Whether experimental tools are enabled for this MCP request or stdio session
- `app.server.distribution` - stdio only: `@sentry/mcp-server` for the npm package, `sentry mcp` for the CLI command
- `app.consent.skill.<skill>.granted` - Per-skill boolean attributes for skills granted to the MCP request. Skill ids are normalized with `-` replaced by `_`
- `app.side_quest.<id>` - `"shown"` or `"capped"` (detected but cut by the cap) for each side quest a tool call detected; unset when not detected. Ids are stable snake_case
- `app.side_quests.detected`, `app.side_quests.shown` - Integer counts of side quests detected and shown; `0` when detectors ran and found none. Each shown quest also increments the `app.side_quest.shown` counter metric, tagged `side_quest.id`, which trace sampling does not drop
- `app.upstream.host` - Upstream Sentry host configured for the server
- `app.url.full` - MCP URL configured for stdio

Expand Down
13 changes: 12 additions & 1 deletion packages/mcp-core/src/internal/issue-side-quests.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import { eventFixture, sourceMapDebugFixture } from "@sentry/mcp-server-mocks";
import { describe, expect, it } from "vitest";
import { EventSchema, SourceMapDebugSchema } from "../api-client/schema";
import { buildIssueSideQuests } from "./issue-side-quests";
import {
ISSUE_SIDE_QUEST_IDS,
buildIssueSideQuests,
} from "./issue-side-quests";

const session = { experimentalMode: false };
const files = ["/a.js", "/b.js", "/c.js"];
Expand Down Expand Up @@ -36,6 +39,14 @@ function eventWithFourFiles() {
}

describe("buildIssueSideQuests", () => {
it("uses unique snake_case ids, which analytics depend on", () => {
const ids = Object.values(ISSUE_SIDE_QUEST_IDS);
expect(new Set(ids).size).toBe(ids.length);
for (const id of ids) {
expect(id).toMatch(/^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$/);
}
});

it("lists each file's cause when the causes differ", () => {
const event = eventWithFourFiles();
const debug = SourceMapDebugSchema.parse(sourceMapDebugFixture);
Expand Down
16 changes: 13 additions & 3 deletions packages/mcp-core/src/internal/issue-side-quests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,16 @@ export interface IssueSideQuestInput {
sourceMapDebug?: SourceMapDebug;
}

/**
* The ids of the issue side quests. Analytics query them as span attributes
* and metric tags, so renaming one breaks dashboards.
*/
export const ISSUE_SIDE_QUEST_IDS = {
jsSourceMaps: "js_source_maps",
missingStackTrace: "missing_stack_trace",
missingUserIdentity: "missing_user_identity",
} as const;

const MAX_FILES = 3;

const JS_SOURCE_MAPS_TROUBLESHOOTING_URL =
Expand Down Expand Up @@ -180,7 +190,7 @@ function detectJsSourceMapFailure({
const fallback = `${matched.map(([, detail]) => detail.problem).join("; ")}. ${[...new Set(matched.map(([, detail]) => detail.fix))].join(" ")}`;
const quest = (tip: string, details?: string[]): SideQuestCandidate => ({
rank: 0,
id: "js_source_maps",
id: ISSUE_SIDE_QUEST_IDS.jsSourceMaps,
tip,
docs: JS_SOURCE_MAPS_TROUBLESHOOTING_URL,
details,
Expand Down Expand Up @@ -314,7 +324,7 @@ function detectMissingUserIdentity({

return {
rank: 10,
id: "missing_user_identity",
id: ISSUE_SIDE_QUEST_IDS.missingUserIdentity,
tip: `${problem} Call ${call} after sign-in; an opaque ID is enough.`,
docs,
};
Expand Down Expand Up @@ -522,7 +532,7 @@ export function hasReadableStack(event: Event): boolean {
}

function missingStackTrace(tip: string, docs: string): SideQuestCandidate {
return { rank: 5, id: "missing_stack_trace", tip, docs };
return { rank: 5, id: ISSUE_SIDE_QUEST_IDS.missingStackTrace, tip, docs };
}

/**
Expand Down
73 changes: 72 additions & 1 deletion packages/mcp-core/src/internal/side-quests.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,20 @@
import { describe, expect, it } from "vitest";
import { getActiveSpan, metrics } from "@sentry/core";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
type SideQuestCandidate,
buildSideQuests,
formatSideQuests,
} from "./side-quests";

vi.mock("@sentry/core", async (importOriginal) => {
const actual = await importOriginal<typeof import("@sentry/core")>();
return {
...actual,
getActiveSpan: vi.fn(),
metrics: { ...actual.metrics, count: vi.fn() },
};
});

const session = {
experimentalMode: false,
availableToolNames: new Set(["get_issue_details", "search_events"]),
Expand Down Expand Up @@ -126,6 +136,67 @@ describe("buildSideQuests", () => {
});
});

describe("side quest telemetry", () => {
const setAttributes = vi.fn();

beforeEach(() => {
setAttributes.mockClear();
vi.mocked(metrics.count).mockClear();
vi.mocked(getActiveSpan).mockReturnValue({
setAttributes,
} as unknown as ReturnType<typeof getActiveSpan>);
});

it("marks each detected quest as shown or capped, with the counts", () => {
buildSideQuests(
null,
[
detector({ rank: 10, id: "late", tip: "Late." }),
detector({ rank: 0, id: "first", tip: "First." }),
detector(null),
detector({ rank: 5, id: "second", tip: "Second." }),
],
session,
2,
);

expect(setAttributes).toHaveBeenCalledOnce();
expect(setAttributes).toHaveBeenCalledWith({
"app.side_quests.detected": 3,
"app.side_quests.shown": 2,
"app.side_quest.first": "shown",
"app.side_quest.second": "shown",
"app.side_quest.late": "capped",
});
});

it("counts each shown quest, not the capped ones", () => {
buildSideQuests(
null,
[
detector({ rank: 0, id: "first", tip: "First." }),
detector({ rank: 1, id: "capped", tip: "Capped." }),
],
session,
1,
);

expect(vi.mocked(metrics.count).mock.calls).toEqual([
["app.side_quest.shown", 1, { attributes: { "side_quest.id": "first" } }],
]);
});

it("records zero counts when no detector finds anything", () => {
buildSideQuests(null, [detector(null)], session);

expect(setAttributes).toHaveBeenCalledWith({
"app.side_quests.detected": 0,
"app.side_quests.shown": 0,
});
expect(metrics.count).not.toHaveBeenCalled();
});
});

describe("formatSideQuests", () => {
it("renders one line per quest, with its calls, docs, and details", () => {
expect(
Expand Down
33 changes: 31 additions & 2 deletions packages/mcp-core/src/internal/side-quests.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { getActiveSpan, metrics } from "@sentry/core";
import { inlineCode } from "@sentry/toolkit-core/markdown";
import { z } from "zod";
import {
Expand Down Expand Up @@ -80,13 +81,18 @@ export function buildSideQuests<Input>(
session: ToolSession,
max = MAX_SIDE_QUESTS,
): SideQuest[] | undefined {
const quests = detectors
const ranked = detectors
.flatMap((detect) => {
const candidate = detect(input);
return candidate ? [candidate] : [];
})
// Array.prototype.sort is stable, so ties keep detector order
.sort((a, b) => a.rank - b.rank)
.sort((a, b) => a.rank - b.rank);
recordSideQuests(
ranked.map(({ id }) => id),
max,
);
const quests = ranked
.slice(0, max)
.map(({ id, tip, next, docs, details }) => {
const calls = (next ?? [])
Expand All @@ -104,6 +110,29 @@ export function buildSideQuests<Input>(
return quests.length > 0 ? quests : undefined;
}

/**
* Records which quests were detected and which the cap cut, so analytics can
* query them without parsing tool output: one `app.side_quest.<id>` span
* attribute per detected quest, the counts, and a metric per shown quest,
* which trace sampling does not drop. Quest ids are part of this contract.
*/
function recordSideQuests(ids: string[], max: number) {
const shown = ids.slice(0, max);
const attributes: Record<string, string | number> = {
"app.side_quests.detected": ids.length,
"app.side_quests.shown": shown.length,
};
ids.forEach((id, index) => {
attributes[`app.side_quest.${id}`] = index < max ? "shown" : "capped";
});
getActiveSpan()?.setAttributes(attributes);
for (const id of shown) {
metrics.count("app.side_quest.shown", 1, {
attributes: { "side_quest.id": id },
});
}
}

/** The `## Concerns Identified` markdown section, or an empty string. */
export function formatSideQuests(quests: SideQuest[] | undefined): string {
if (!quests) {
Expand Down
1 change: 1 addition & 0 deletions packages/mcp-core/src/server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ vi.mock("@sentry/core", () => ({
captureException: vi.fn(),
captureMessage: vi.fn(),
getClient: vi.fn(),
metrics: { count: vi.fn() },
}));

vi.mock("@sentry/core/server", () => ({
Expand Down
Loading