Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/operations/monitoring.md
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ These are Sentry MCP application attributes that are not part of the MCP semanti
- `app.consent.skill.<skill>.granted` - Per-skill boolean attributes for skills granted to the MCP request. Skill ids are normalized with `-` replaced by `_`
- `app.side_quest.<id>` - `"shown"` or `"capped"` (detected but cut by the cap) for each side quest a tool call detected; unset when not detected. Ids are stable snake_case
- `app.side_quests.detected`, `app.side_quests.shown` - Integer counts of side quests detected and shown; `0` when detectors ran and found none. Each shown quest also increments the `app.side_quest.shown` counter metric, tagged `side_quest.id`, which trace sampling does not drop
- `app.side_quest.<id>.next` - Comma-separated fingerprints of the `next` calls a shown side quest offered; unset when it offered none
- `app.tool_call.fingerprint` - Fingerprint of the tool call: a SHA-256 hash (16 hex characters) of the tool name and its arguments, without scope keys (organization, project, region) or empty values. For `execute_sentry_tool`, it is set on the child span of the tool it runs. A span from the same user whose fingerprint is in an earlier span's `app.side_quest.<id>.next` is a call that quest offered. Both spans are sampled independently, so divide the matched share by the sample rate
- `app.side_quest.related_issues_open.issue_ids` - Comma-separated short IDs of the related issues that `update_issue`'s `related_issues_open` quest lists (up to 3), so analytics can match a later `get_sentry_resource` or `update_issue` call to a suggested issue
- `app.side_quest.related_issues_open.count` - Integer count of related issues still open when `update_issue` shows the `related_issues_open` quest, the number its tip states; a lower bound when a relation type filled its page
- `app.upstream.host` - Upstream Sentry host configured for the server
Expand Down
67 changes: 67 additions & 0 deletions packages/mcp-core/src/internal/side-quests.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import {
type SideQuestCandidate,
buildSideQuests,
fingerprintToolCall,
formatSideQuests,
} from "./side-quests";

Expand Down Expand Up @@ -186,6 +187,35 @@ describe("side quest telemetry", () => {
]);
});

it("records the fingerprints of the calls each shown quest offered", () => {
buildSideQuests(
null,
[
detector({
rank: 0,
id: "calls",
tip: "Tip.",
next: [
{ tool: "update_project", args: { slug: "web" } },
{ tool: "get_issue_details", args: { issueId: "WEB-1" } },
{ tool: "search_events", args: { query: "a" } },
],
}),
detector({ rank: 1, id: "tip_only", tip: "Tip." }),
],
session,
);

// update_project is not in the session, so only the calls the quest
// shows are recorded.
expect(setAttributes).toHaveBeenLastCalledWith({
"app.side_quest.calls.next": [
fingerprintToolCall("get_issue_details", { issueId: "WEB-1" }),
fingerprintToolCall("search_events", { query: "a" }),
].join(","),
});
});

it("records zero counts when no detector finds anything", () => {
buildSideQuests(null, [detector(null)], session);

Expand All @@ -197,6 +227,43 @@ describe("side quest telemetry", () => {
});
});

describe("fingerprintToolCall", () => {
const call = {
organizationSlug: "acme",
resourceType: "issue",
resourceId: "WEB-1",
};

it("is a short hex id", () => {
expect(fingerprintToolCall("get_sentry_resource", call)).toMatch(
/^[0-9a-f]{16}$/,
);
});

it("ignores key order, scope keys, and empty values", () => {
expect(
fingerprintToolCall("get_sentry_resource", {
resourceId: "WEB-1",
regionUrl: "https://us.sentry.io",
resourceType: "issue",
url: "",
packageNames: undefined,
}),
).toBe(fingerprintToolCall("get_sentry_resource", call));
});

it("differs by tool and by argument value", () => {
const fingerprint = fingerprintToolCall("get_sentry_resource", call);
expect(fingerprintToolCall("update_issue", call)).not.toBe(fingerprint);
expect(
fingerprintToolCall("get_sentry_resource", {
...call,
resourceId: "WEB-2",
}),
).not.toBe(fingerprint);
});
});

describe("formatSideQuests", () => {
it("renders one line per quest, with its calls, docs, and details", () => {
expect(
Expand Down
53 changes: 53 additions & 0 deletions packages/mcp-core/src/internal/side-quests.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { createHash } from "node:crypto";
import { getActiveSpan, metrics } from "@sentry/core";
import { inlineCode } from "@sentry/toolkit-core/markdown";
import { z } from "zod";
Expand All @@ -7,6 +8,7 @@ import {
getToolCallRoute,
} from "./tool-helpers/tool-call-formatting";
import type { SuggestedActionValue } from "./tool-helpers/suggested-actions";
import { CONSTRAINT_PARAMETER_KEYS } from "../types";

/**
* Side quests are optional detours a tool response points out, such as a
Expand Down Expand Up @@ -107,9 +109,60 @@ export function buildSideQuests<Input>(
...(lines.length > 0 && { details: lines }),
};
});
recordOfferedCalls(quests);
return quests.length > 0 ? quests : undefined;
}

/** The span attribute every tool call records its fingerprint in. */
export const TOOL_CALL_FINGERPRINT_ATTRIBUTE = "app.tool_call.fingerprint";

/**
* A short, stable id for a tool call: the tool name and its arguments in key
* order, hashed with SHA-256 and cut to 16 hex characters. A shown quest
* records the fingerprints of its `next` calls, and every tool call records
* its own, so analytics can tell when a later call from the same user is one
* a quest offered.
*
* Scope keys (organization, project, region) are left out: the server fills
* them from constraints or the default organization, so an agent may or may
* not pass them. Empty values are left out for the same reason.
*/
export function fingerprintToolCall(
tool: string,
args: Record<string, unknown>,
): string {
const entries = Object.entries(args)
.filter(
([key, value]) =>
!CONSTRAINT_PARAMETER_KEYS.has(key) && value != null && value !== "",
)
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
return createHash("sha256")
.update(JSON.stringify([tool, entries]))
.digest("hex")
.slice(0, 16);
}

/**
* Records the `next` calls each shown quest offered as
* `app.side_quest.<id>.next`, a comma-separated list of fingerprints. A later
* span from the same user whose {@link TOOL_CALL_FINGERPRINT_ATTRIBUTE} is in
* that list is a call the quest offered.
*/
function recordOfferedCalls(quests: SideQuest[]) {
const attributes: Record<string, string> = {};
for (const { id, next } of quests) {
if (next?.length) {
attributes[`app.side_quest.${id}.next`] = next
.map(({ tool, args }) => fingerprintToolCall(tool, args))
.join(",");
}
}
if (Object.keys(attributes).length > 0) {
getActiveSpan()?.setAttributes(attributes);
Comment on lines +146 to +162

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if next is specifically for tools then how will we track non-tool calling side quests? For context this is what the related_issues sidequest curr shows up as in mcp responses

CommitRepository: "getsentry/sentry")
  ⎿  {
       "organizationSlug": "sentry",
       "issueId": "SENTRY-60NJ",
       "title": "IntegrationError: invalid_blocks",
       "issueUrl": "https://sentry.sentry.io/issues/SENTRY-60NJ",
       "updated": true,
       "changes": {
         "status": {
           "from": "unresolved",
           "to": "resolvedInCommit"
         },
         "ignoreBehavior": null,
         "assignedTo": null,
         "priority": null
       },
       "current": {
         "status": "resolvedInCommit",
         "ignoreBehavior": null,
         "assignedTo": "issue-management",
         "priority": "high"
       },
       "effect": "The issue is now marked as resolved in commit c206d1d6ce476260680ca1e19e93422d422cff2d. It will be resolved in the first release that contains this
     commit",
       "reasonComment": null,
       "sideQuests": [
         {
           "id": "related_issues_open",
           "tip": "1 issue related to this one is still open. Check whether this change covers it too.",
           "next": [
             {
               "tool": "get_sentry_resource",
               "args": {
                 "organizationSlug": "sentry",
                 "resourceType": "issue",
                 "resourceId": "SENTRY-5PCN"
               }
             }
           ],
           "details": [
             "`SENTRY-5PCN` (similar stack trace): `IntegrationError: invalid_blocks`"
           ]
         }
       ]
     }

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and then agents do some decision making (there was no additional user ask to me)

I've resolved SENTRY-60NJ. A similar issue, SENTRY-5PCN, is still open, so I'm checking whether your fix covers it too.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, this only covers quests that have next. Tip-only quests (js_source_maps, missing_user_identity, missing_stack_trace) have nothing to match, because the fix happens in the user's code, outside MCP. For those we'd have to look at outcomes instead, like the quest no longer showing up for that project.

Your example is the case this is built for. The agent's next call, get_sentry_resource(resourceType='issue', resourceId='SENTRY-5PCN'), hashes to the same fingerprint as the next entry, so it counts as taken. If the agent skipped that and went straight to update_issue on SENTRY-5PCN, the fingerprint wouldn't match, but #2387's issue_ids attribute still catches it.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, that's exactly what we want to count. The user didn't have to ask: the agent followed the quest on its own. Its get_sentry_resource call for SENTRY-5PCN matches the offered call, so it shows up as taken.

}
}

/**
* Records which quests were detected and which the cap cut, so analytics can
* query them without parsing tool output: one `app.side_quest.<id>` span
Expand Down
36 changes: 36 additions & 0 deletions packages/mcp-core/src/server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { McpServer } from "@modelcontextprotocol/server";
import {
type Span,
captureException,
getActiveSpan,
getClient,
setUser,
startSpan,
Expand All @@ -22,6 +23,7 @@ import { z } from "zod";
import { MAX_TOOL_INPUT_ELEMENTS } from "@sentry/toolkit-core/mcp-limits";
import { ApiAuthenticationError } from "./api-client";
import { UserInputError } from "./errors";
import { fingerprintToolCall } from "./internal/side-quests";
import { apiServiceFromContext } from "./internal/tool-helpers/api";
import { structuredResult } from "./internal/tool-helpers/results";
import { buildServer } from "./server";
Expand Down Expand Up @@ -3277,6 +3279,40 @@ describe("buildServer", () => {
expect(span?.recordException).not.toHaveBeenCalled();
});

it("records each call's fingerprint on the span of the tool that ran", async () => {
const registry: Record<string, ToolConfig> = {};
registry["fake_catalog_tool"] = createMockTool("fake_catalog_tool", {
inputSchema: { value: z.string() },
});
registry["execute_sentry_tool"] = createExecuteTool(() => registry);
const activeSpan = { ...createMockSpan(), setAttributes: vi.fn() };
vi.mocked(getActiveSpan).mockReturnValue(activeSpan as unknown as Span);
const server = buildServer({ context: baseContext, tools: registry });
const fingerprint = fingerprintToolCall("fake_catalog_tool", {
value: "a",
});

await callRegisteredTool(server, "execute_sentry_tool", {
name: "fake_catalog_tool",
arguments: { value: "a" },
});
expect(startedSpans[0]?.setAttribute).toHaveBeenCalledWith(
"app.tool_call.fingerprint",
fingerprint,
);
expect(activeSpan.setAttribute).not.toHaveBeenCalledWith(
"app.tool_call.fingerprint",
expect.anything(),
);

await callRegisteredTool(server, "fake_catalog_tool", { value: "a" });
expect(activeSpan.setAttribute).toHaveBeenCalledWith(
"app.tool_call.fingerprint",
fingerprint,
);
vi.mocked(getActiveSpan).mockReturnValue(undefined);
});

it("execute_sentry_tool injects constrained arguments for catalog-only tools", async () => {
const server = buildServer({
context: {
Expand Down
12 changes: 12 additions & 0 deletions packages/mcp-core/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ import {
isExpectedToolError,
recordToolFailure,
} from "./internal/error-handling";
import {
TOOL_CALL_FINGERPRINT_ATTRIBUTE,
fingerprintToolCall,
} from "./internal/side-quests";
import type { Skill } from "./skills";
import { type LogIssueOptions, logIssue } from "./telem/logging";
import {
Expand Down Expand Up @@ -459,6 +463,14 @@ function configureServer({
);

if (activeSpan) {
// execute_sentry_tool records the fingerprint of the tool it runs
// on its own child span.
if (tool.name !== "execute_sentry_tool") {
activeSpan.setAttribute(
TOOL_CALL_FINGERPRINT_ATTRIBUTE,
fingerprintToolCall(tool.name, paramsWithConstraints),
);
}
// Intentional GenAI semconv extension: per-key attrs like http.request.header.<key>.
for (const [key, value] of Object.entries(paramsWithConstraints)) {
const attributeValue =
Expand Down
8 changes: 8 additions & 0 deletions packages/mcp-core/src/tools/special/execute-tool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ import {
recordToolFailure,
} from "../../internal/error-handling";
import { UserInputError } from "../../errors";
import {
TOOL_CALL_FINGERPRINT_ATTRIBUTE,
fingerprintToolCall,
} from "../../internal/side-quests";
import { withRoutedCredential } from "../../internal/tool-helpers/credential-routing";
import { ALL_SKILLS } from "../../skills";
import type { ServerContext } from "../../types";
Expand Down Expand Up @@ -66,6 +70,10 @@ async function executeCatalogToolWithSpan({
context,
});
setToolArgumentAttributes(span, paramsWithConstraints);
span.setAttribute(
TOOL_CALL_FINGERPRINT_ATTRIBUTE,
fingerprintToolCall(tool.name, paramsWithConstraints),
);
const output = await withRoutedCredential(
context,
paramsWithConstraints,
Expand Down
Loading