From 8e1069ef555195a6d88e43d58fc1b4b4a598f9b5 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 10 Aug 2026 10:20:57 -0400 Subject: [PATCH 1/2] fix(claude-agent-sdk): stop assigning query usage to the final call `ClaudeAgentSDKPlugin` copied session usage from the terminal result event to the final assistant message in each `query()`. The final `anthropic.messages.create` span then combined request input with cache totals for the full session. ```text expected final call: prompt = 100 input + 65,000 cache read + 2,000 cache creation = 67,100 completion = 1,000 previous final span: prompt = request input + session cache read and creation = 242,100 completion = session output - earlier output snapshots = 3,497 ``` Assistant messages contain the initial `message_start` output count, not the final output count. The plugin subtracted these initial counts from the session output. This calculation assigned all remaining output to the final request. It did not calculate the output for each request. The plugin also changed an SDK message that the caller could read. This defect caused incorrect prompt tokens, completion tokens, and token-based cost. Prompt caching is enabled by default, so the defect affects all JavaScript Claude Agent SDK users. Both `wrapClaudeAgentSDK` and automatic instrumentation use the same plugin. The defect was present from version 3.21.0 through the previous main branch. It is the JavaScript equivalent of Python issue SDK-52. The caller's `includePartialMessages` option now determines the source of final per-request usage. Braintrust keeps this option unchanged because enabling it would add public stream events. ```text includePartialMessages: true message_start -> request ID and per-request prompt/cache usage message_delta -> final per-request completion usage result -> query metadata only includePartialMessages: false or omitted (Claude SDK default) assistant -> request ID and per-request prompt/cache usage transcript -> final completion usage, when an exact match exists no match -> omit completion_tokens and tokens result -> query metadata only ``` When the option is `true`, the caller still receives all original partial events. The plugin merges each final stream update with the assistant usage for the same message ID. A partial update cannot remove valid prompt or cache usage. When the option is `false` or omitted, the caller does not receive partial events. The plugin ignores the initial output snapshot. Prompt and cache metrics remain exact because the assistant message reports them for one request. Completion and total metrics are exact only when transcript recovery succeeds. An unavailable transcript causes missing completion metrics, not zero or estimated metrics. The plugin gets transcript paths from passive SDK hooks. It keeps the root path separate from each subagent path. ```text SessionStart ------\ UserPromptSubmit ---+--> root transcript path ------> [root, message ID] SessionEnd --------/ SubagentStop(tool A) ---> transcript path A --------> [tool A, message ID] SubagentStop(tool B) ---> transcript path B --------> [tool B, message ID] ``` The agent key and message ID form one lookup key. This key prevents a root message from matching a subagent row. It also keeps two subagents separate when their streams run at the same time. ```text Claude SDK Plugin Transcript assistant(id=msg_1) --------> save [root, msg_1] and span query stream ends ----------> read root path -------------> search backward for msg_1 <----------------------------- newest valid usage row update span [root, msg_1] missing row ----------------> wait 25 ms and read again maximum: 3 reads ``` The reader searches only for pending message IDs. It selects the newest valid row because a transcript can contain multiple snapshots for one message. It does not parse every row in a large resumed session. Transcript errors do not affect the query. The retries can add at most 50 ms after stream completion. The plugin does not store transcript content or transcript paths in span data. The plugin does not record terminal usage on the root task span. Braintrust summaries already add the metrics from model-call spans. A second copy on the root span would count the same tokens and cost twice. The Anthropic instrumentation now reads the 5-minute and 1-hour cache-write fields. It uses the object returned by `finalizeAnthropicTokens`. Therefore, each span contains the TTL fields or the legacy aggregate field, but not both. Prompt totals use the same cache representation that the span stores. Unit and end-to-end tests compare each model-call span with stream or transcript usage. Tests also cover both partial-message settings, missing transcripts, unchanged SDK messages, null usage fields, wrappers, and automatic instrumentation. --- .changeset/claude-agent-token-accounting.md | 5 + ...thropic-bedrock-v0-auto-cjs.span-tree.json | 6 +- ...nthropic-bedrock-v0-auto-cjs.span-tree.txt | 6 +- ...thropic-bedrock-v0-auto-esm.span-tree.json | 6 +- ...nthropic-bedrock-v0-auto-esm.span-tree.txt | 6 +- ...-bedrock-v0-latest-auto-cjs.span-tree.json | 6 +- ...c-bedrock-v0-latest-auto-cjs.span-tree.txt | 6 +- ...-bedrock-v0-latest-auto-esm.span-tree.json | 6 +- ...c-bedrock-v0-latest-auto-esm.span-tree.txt | 6 +- ...c-bedrock-v0-latest-wrapped.span-tree.json | 6 +- ...ic-bedrock-v0-latest-wrapped.span-tree.txt | 6 +- ...nthropic-bedrock-v0-wrapped.span-tree.json | 6 +- ...anthropic-bedrock-v0-wrapped.span-tree.txt | 6 +- ...anthropic-v0-latest-wrapped.span-tree.json | 51 +- .../anthropic-v0-latest-wrapped.span-tree.txt | 51 +- .../anthropic-v0-latest.span-tree.json | 51 +- .../anthropic-v0-latest.span-tree.txt | 51 +- .../__snapshots__/anthropic-v0.span-tree.json | 24 +- .../__snapshots__/anthropic-v0.span-tree.txt | 24 +- ...aude-agent-sdk-v0-auto-hook.span-tree.json | 123 ++-- ...laude-agent-sdk-v0-auto-hook.span-tree.txt | 123 ++-- ...ent-sdk-v0-latest-auto-hook.span-tree.json | 126 ++-- ...gent-sdk-v0-latest-auto-hook.span-tree.txt | 126 ++-- ...agent-sdk-v0-latest-wrapped.span-tree.json | 126 ++-- ...-agent-sdk-v0-latest-wrapped.span-tree.txt | 126 ++-- ...claude-agent-sdk-v0-wrapped.span-tree.json | 123 ++-- .../claude-agent-sdk-v0-wrapped.span-tree.txt | 123 ++-- .../assertions.ts | 306 +++++++- .../cassette-filter.mjs | 20 +- .../scenario.impl.mjs | 285 +++++++- .../plugins/anthropic-plugin.test.ts | 48 ++ .../plugins/anthropic-plugin.ts | 18 + .../plugins/claude-agent-sdk-plugin.test.ts | 459 +++++++++++- .../plugins/claude-agent-sdk-plugin.ts | 665 ++++++++++++++++-- .../plugins/github-copilot-plugin.ts | 10 +- js/src/vendor-sdk-types/anthropic.ts | 11 + js/src/vendor-sdk-types/claude-agent-sdk.ts | 30 +- .../ai-sdk/deprecated/BraintrustMiddleware.ts | 5 +- js/src/wrappers/anthropic-tokens-util.test.ts | 68 ++ js/src/wrappers/anthropic-tokens-util.ts | 44 +- 40 files changed, 2663 insertions(+), 631 deletions(-) create mode 100644 .changeset/claude-agent-token-accounting.md create mode 100644 js/src/wrappers/anthropic-tokens-util.test.ts diff --git a/.changeset/claude-agent-token-accounting.md b/.changeset/claude-agent-token-accounting.md new file mode 100644 index 000000000..23d727d30 --- /dev/null +++ b/.changeset/claude-agent-token-accounting.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(claude-agent-sdk): Correct per-call token and cost metrics diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json index cb3b13064..1a4e1fd9c 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -98,7 +99,8 @@ }, "metrics": { "completion_tokens": 6, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 30, "time_to_first_token": 0, @@ -143,7 +145,8 @@ }, "metrics": { "completion_tokens": 7, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 15, "time_to_first_token": 0, @@ -206,7 +209,8 @@ }, "metrics": { "completion_tokens": 27, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "time_to_first_token": 0, @@ -245,7 +249,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -284,7 +289,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -358,7 +364,8 @@ }, "metrics": { "completion_tokens": 26, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -428,7 +435,8 @@ }, "metrics": { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 589, "time_to_first_token": 0, @@ -580,7 +588,8 @@ }, "metrics": { "completion_tokens": 77, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 17662, "server_tool_use_web_fetch_requests": 0, @@ -638,7 +647,8 @@ }, "metrics": { "completion_tokens": 48, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 49, "time_to_first_token": 0, @@ -683,7 +693,8 @@ }, "metrics": { "completion_tokens": 5, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, @@ -722,7 +733,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -761,7 +773,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -835,7 +848,8 @@ }, "metrics": { "completion_tokens": 26, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -910,7 +924,8 @@ }, "metrics": { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 607, "time_to_first_token": 0, @@ -1009,7 +1024,8 @@ }, "metrics": { "completion_tokens": 16, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -1048,7 +1064,8 @@ }, "metrics": { "completion_tokens": 72, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1294, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt index b2830b010..f9d451a62 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -89,7 +90,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 6, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 30, │ "time_to_first_token": 0, @@ -126,7 +128,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 7, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 15, │ "time_to_first_token": 0, @@ -181,7 +184,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 27, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 1389, │ "time_to_first_token": 0, @@ -212,7 +216,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -243,7 +248,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -309,7 +315,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 26, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, @@ -371,7 +378,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 56, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 589, │ "time_to_first_token": 0, @@ -515,7 +523,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 77, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 17662, │ "server_tool_use_web_fetch_requests": 0, @@ -565,7 +574,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 48, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 49, │ "time_to_first_token": 0, @@ -602,7 +612,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 5, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, @@ -633,7 +644,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -664,7 +676,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -730,7 +743,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 26, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, @@ -773,7 +787,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 72, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 1294, │ "time_to_first_token": 0, @@ -831,7 +846,8 @@ span_tree: │ │ } │ │ metrics: { │ │ "completion_tokens": 56, - │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 0, │ │ "prompt_tokens": 607, │ │ "time_to_first_token": 0, @@ -918,7 +934,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 16, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json index cb3b13064..1a4e1fd9c 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -98,7 +99,8 @@ }, "metrics": { "completion_tokens": 6, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 30, "time_to_first_token": 0, @@ -143,7 +145,8 @@ }, "metrics": { "completion_tokens": 7, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 15, "time_to_first_token": 0, @@ -206,7 +209,8 @@ }, "metrics": { "completion_tokens": 27, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "time_to_first_token": 0, @@ -245,7 +249,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -284,7 +289,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -358,7 +364,8 @@ }, "metrics": { "completion_tokens": 26, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -428,7 +435,8 @@ }, "metrics": { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 589, "time_to_first_token": 0, @@ -580,7 +588,8 @@ }, "metrics": { "completion_tokens": 77, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 17662, "server_tool_use_web_fetch_requests": 0, @@ -638,7 +647,8 @@ }, "metrics": { "completion_tokens": 48, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 49, "time_to_first_token": 0, @@ -683,7 +693,8 @@ }, "metrics": { "completion_tokens": 5, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, @@ -722,7 +733,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -761,7 +773,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -835,7 +848,8 @@ }, "metrics": { "completion_tokens": 26, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -910,7 +924,8 @@ }, "metrics": { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 607, "time_to_first_token": 0, @@ -1009,7 +1024,8 @@ }, "metrics": { "completion_tokens": 16, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -1048,7 +1064,8 @@ }, "metrics": { "completion_tokens": 72, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1294, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt index b2830b010..f9d451a62 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -89,7 +90,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 6, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 30, │ "time_to_first_token": 0, @@ -126,7 +128,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 7, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 15, │ "time_to_first_token": 0, @@ -181,7 +184,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 27, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 1389, │ "time_to_first_token": 0, @@ -212,7 +216,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -243,7 +248,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -309,7 +315,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 26, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, @@ -371,7 +378,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 56, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 589, │ "time_to_first_token": 0, @@ -515,7 +523,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 77, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 17662, │ "server_tool_use_web_fetch_requests": 0, @@ -565,7 +574,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 48, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 49, │ "time_to_first_token": 0, @@ -602,7 +612,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 5, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, @@ -633,7 +644,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -664,7 +676,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -730,7 +743,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 26, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, @@ -773,7 +787,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 72, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 1294, │ "time_to_first_token": 0, @@ -831,7 +846,8 @@ span_tree: │ │ } │ │ metrics: { │ │ "completion_tokens": 56, - │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 0, │ │ "prompt_tokens": 607, │ │ "time_to_first_token": 0, @@ -918,7 +934,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 16, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.json index 7f6197d6b..a6dbf0e8c 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -98,7 +99,8 @@ }, "metrics": { "completion_tokens": 6, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 30, "time_to_first_token": 0, @@ -143,7 +145,8 @@ }, "metrics": { "completion_tokens": 7, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 15, "time_to_first_token": 0, @@ -206,7 +209,8 @@ }, "metrics": { "completion_tokens": 27, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "time_to_first_token": 0, @@ -245,7 +249,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -284,7 +289,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -358,7 +364,8 @@ }, "metrics": { "completion_tokens": 26, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -428,7 +435,8 @@ }, "metrics": { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 589, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.txt index 2c194b682..e977dda70 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -89,7 +90,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 6, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 30, │ "time_to_first_token": 0, @@ -126,7 +128,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 7, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 15, │ "time_to_first_token": 0, @@ -181,7 +184,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 27, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 1389, │ "time_to_first_token": 0, @@ -212,7 +216,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -243,7 +248,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -309,7 +315,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 26, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, @@ -371,7 +378,8 @@ span_tree: } metrics: { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 589, "time_to_first_token": 0, diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json index 8126cd762..020a98518 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json @@ -52,13 +52,15 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 18783, + "completion_tokens": 174, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18783, "prompt_tokens": 18793, - "tokens": 18796 + "tokens": 18967 } }, { @@ -140,14 +142,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 216, - "prompt_cache_creation_tokens": 18988, + "completion_tokens": 45, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 205, "prompt_cached_tokens": 18783, - "prompt_tokens": 37779, - "tokens": 37995 + "prompt_tokens": 18996, + "tokens": 19041 } } ], @@ -217,11 +221,13 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { "completion_tokens": 171, - "prompt_cache_creation_tokens": 18650, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18650, "prompt_tokens": 18660, "tokens": 18831 } @@ -271,14 +277,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 19, - "prompt_cache_creation_tokens": 184, + "completion_tokens": 190, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 184, "prompt_cached_tokens": 18650, "prompt_tokens": 18844, - "tokens": 18863 + "tokens": 19034 } } ], @@ -369,13 +377,15 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 18811, + "completion_tokens": 215, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18811, "prompt_tokens": 18821, - "tokens": 18824 + "tokens": 19036 } }, { @@ -422,13 +432,15 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 26759, + "completion_tokens": 106, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 26759, "prompt_tokens": 26762, - "tokens": 26765 + "tokens": 26868 } }, { @@ -577,14 +589,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 291, - "prompt_cache_creation_tokens": 19118, + "completion_tokens": 82, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 307, "prompt_cached_tokens": 18811, - "prompt_tokens": 37937, - "tokens": 38228 + "prompt_tokens": 19126, + "tokens": 19208 } } ], @@ -662,13 +676,15 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 4, - "prompt_cache_creation_tokens": 18711, + "completion_tokens": 169, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18711, "prompt_tokens": 18721, - "tokens": 18725 + "tokens": 18890 } }, { @@ -714,13 +730,15 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 50, - "prompt_cache_creation_tokens": 17229, + "completion_tokens": 76, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 17229, "prompt_tokens": 17232, - "tokens": 17282 + "tokens": 17308 } }, { @@ -863,14 +881,16 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 281, - "prompt_cache_creation_tokens": 18961, + "completion_tokens": 166, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 250, "prompt_cached_tokens": 18711, - "prompt_tokens": 37680, - "tokens": 37961 + "prompt_tokens": 18969, + "tokens": 19135 } } ], @@ -949,13 +969,14 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 8, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 18784, - "prompt_tokens": 18794, - "tokens": 18802 + "prompt_tokens": 18794 } }, { @@ -1031,14 +1052,14 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 225, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 37568, - "prompt_tokens": 37775, - "tokens": 38000 + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 18784, + "prompt_tokens": 18991 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt index 581af7a63..eaa5b4c46 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt @@ -64,13 +64,15 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cache_creation_tokens": 18783, + │ │ "completion_tokens": 174, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 18783, │ │ "prompt_tokens": 18793, - │ │ "tokens": 18796 + │ │ "tokens": 18967 │ │ } │ ├── tool: calculator/calculator [tool] │ │ input: { @@ -140,14 +142,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 216, - │ "prompt_cache_creation_tokens": 18988, + │ "completion_tokens": 45, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 205, │ "prompt_cached_tokens": 18783, - │ "prompt_tokens": 37779, - │ "tokens": 37995 + │ "prompt_tokens": 18996, + │ "tokens": 19041 │ } ├── claude-agent-async-prompt-operation │ metadata: { @@ -220,11 +224,13 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { │ │ "completion_tokens": 171, - │ │ "prompt_cache_creation_tokens": 18650, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 18650, │ │ "prompt_tokens": 18660, │ │ "tokens": 18831 │ │ } @@ -270,14 +276,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 19, - │ "prompt_cache_creation_tokens": 184, + │ "completion_tokens": 190, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 184, │ "prompt_cached_tokens": 18650, │ "prompt_tokens": 18844, - │ "tokens": 18863 + │ "tokens": 19034 │ } ├── claude-agent-subagent-operation │ metadata: { @@ -342,13 +350,15 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cache_creation_tokens": 18811, + │ │ "completion_tokens": 215, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 18811, │ │ "prompt_tokens": 18821, - │ │ "tokens": 18824 + │ │ "tokens": 19036 │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -448,13 +458,15 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ │ metrics: { - │ │ │ "completion_tokens": 3, - │ │ │ "prompt_cache_creation_tokens": 26759, + │ │ │ "completion_tokens": 106, + │ │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ │ "prompt_cache_creation_5m_tokens": 26759, │ │ │ "prompt_tokens": 26762, - │ │ │ "tokens": 26765 + │ │ │ "tokens": 26868 │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { @@ -524,14 +536,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 291, - │ "prompt_cache_creation_tokens": 19118, + │ "completion_tokens": 82, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 307, │ "prompt_cached_tokens": 18811, - │ "prompt_tokens": 37937, - │ "tokens": 38228 + │ "prompt_tokens": 19126, + │ "tokens": 19208 │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { @@ -597,13 +611,15 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-sonnet-4-5-20250929" + │ │ "model": "claude-sonnet-4-5-20250929", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 4, - │ │ "prompt_cache_creation_tokens": 18711, + │ │ "completion_tokens": 169, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 18711, │ │ "prompt_tokens": 18721, - │ │ "tokens": 18725 + │ │ "tokens": 18890 │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -702,13 +718,15 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ │ metrics: { - │ │ │ "completion_tokens": 50, - │ │ │ "prompt_cache_creation_tokens": 17229, + │ │ │ "completion_tokens": 76, + │ │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ │ "prompt_cache_creation_5m_tokens": 17229, │ │ │ "prompt_tokens": 17232, - │ │ │ "tokens": 17282 + │ │ │ "tokens": 17308 │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { @@ -776,14 +794,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-sonnet-4-5-20250929" + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 281, - │ "prompt_cache_creation_tokens": 18961, + │ "completion_tokens": 166, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 250, │ "prompt_cached_tokens": 18711, - │ "prompt_tokens": 37680, - │ "tokens": 37961 + │ "prompt_tokens": 18969, + │ "tokens": 19135 │ } └── claude-agent-failure-operation metadata: { @@ -845,13 +865,14 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 8, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 18784, - │ "prompt_tokens": 18794, - │ "tokens": 18802 + │ "prompt_tokens": 18794 │ } ├── tool: calculator/calculator [tool] │ input: { @@ -915,12 +936,12 @@ span_tree: } ] metadata: { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } metrics: { - "completion_tokens": 225, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 37568, - "prompt_tokens": 37775, - "tokens": 38000 + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 18784, + "prompt_tokens": 18991 } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json index 79253c91e..54de1f69c 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json @@ -52,13 +52,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 7, + "completion_tokens": 168, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 24243, "prompt_tokens": 24253, - "tokens": 24260 + "tokens": 24421 } }, { @@ -140,14 +143,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 209, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 48486, - "prompt_tokens": 48693, - "tokens": 48902 + "completion_tokens": 48, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 24243, + "prompt_tokens": 24450, + "tokens": 24498 } } ], @@ -217,10 +222,13 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { "completion_tokens": 475, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 24110, "prompt_tokens": 24120, "tokens": 24595 @@ -271,14 +279,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 4, - "prompt_cache_creation_tokens": 488, + "completion_tokens": 148, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 488, "prompt_cached_tokens": 24110, "prompt_tokens": 24608, - "tokens": 24612 + "tokens": 24756 } } ], @@ -369,13 +379,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, + "completion_tokens": 234, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 24271, "prompt_tokens": 24281, - "tokens": 24284 + "tokens": 24515 } }, { @@ -422,14 +435,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 2201, + "completion_tokens": 106, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2201, "prompt_cached_tokens": 16794, "prompt_tokens": 18998, - "tokens": 19001 + "tokens": 19104 } }, { @@ -600,14 +615,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 288, - "prompt_cache_creation_tokens": 331, - "prompt_cached_tokens": 48542, - "prompt_tokens": 48881, - "tokens": 49169 + "completion_tokens": 60, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 331, + "prompt_cached_tokens": 24271, + "prompt_tokens": 24610, + "tokens": 24670 } } ], @@ -685,13 +702,16 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, + "completion_tokens": 259, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 24171, "prompt_tokens": 24181, - "tokens": 24184 + "tokens": 24440 } }, { @@ -737,14 +757,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 57, - "prompt_cache_creation_tokens": 1359, + "completion_tokens": 74, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 1359, "prompt_cached_tokens": 4548, "prompt_tokens": 5910, - "tokens": 5967 + "tokens": 5984 } }, { @@ -909,14 +931,16 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 308, - "prompt_cache_creation_tokens": 353, - "prompt_cached_tokens": 48342, - "prompt_tokens": 48703, - "tokens": 49011 + "completion_tokens": 109, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 353, + "prompt_cached_tokens": 24171, + "prompt_tokens": 24532, + "tokens": 24641 } } ], @@ -995,14 +1019,14 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 7, - "prompt_cache_creation_tokens": 2374, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2374, "prompt_cached_tokens": 21870, - "prompt_tokens": 24254, - "tokens": 24261 + "prompt_tokens": 24254 } }, { @@ -1078,14 +1102,14 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 260, - "prompt_cache_creation_tokens": 2605, - "prompt_cached_tokens": 46114, - "prompt_tokens": 48727, - "tokens": 48987 + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 231, + "prompt_cached_tokens": 24244, + "prompt_tokens": 24483 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt index 3cd64950c..17c40ead5 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt @@ -64,13 +64,16 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 7, + │ │ "completion_tokens": 168, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 24243, │ │ "prompt_tokens": 24253, - │ │ "tokens": 24260 + │ │ "tokens": 24421 │ │ } │ ├── tool: calculator/calculator [tool] │ │ input: { @@ -140,14 +143,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 209, - │ "prompt_cache_creation_tokens": 199, - │ "prompt_cached_tokens": 48486, - │ "prompt_tokens": 48693, - │ "tokens": 48902 + │ "completion_tokens": 48, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 199, + │ "prompt_cached_tokens": 24243, + │ "prompt_tokens": 24450, + │ "tokens": 24498 │ } ├── claude-agent-async-prompt-operation │ metadata: { @@ -220,10 +225,13 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { │ │ "completion_tokens": 475, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 24110, │ │ "prompt_tokens": 24120, │ │ "tokens": 24595 @@ -270,14 +278,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 488, + │ "completion_tokens": 148, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 488, │ "prompt_cached_tokens": 24110, │ "prompt_tokens": 24608, - │ "tokens": 24612 + │ "tokens": 24756 │ } ├── claude-agent-subagent-operation │ metadata: { @@ -342,13 +352,16 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 3, + │ │ "completion_tokens": 234, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 24271, │ │ "prompt_tokens": 24281, - │ │ "tokens": 24284 + │ │ "tokens": 24515 │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -470,14 +483,16 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ │ metrics: { - │ │ │ "completion_tokens": 3, - │ │ │ "prompt_cache_creation_tokens": 2201, + │ │ │ "completion_tokens": 106, + │ │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ │ "prompt_cache_creation_5m_tokens": 2201, │ │ │ "prompt_cached_tokens": 16794, │ │ │ "prompt_tokens": 18998, - │ │ │ "tokens": 19001 + │ │ │ "tokens": 19104 │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { @@ -547,14 +562,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 288, - │ "prompt_cache_creation_tokens": 331, - │ "prompt_cached_tokens": 48542, - │ "prompt_tokens": 48881, - │ "tokens": 49169 + │ "completion_tokens": 60, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 331, + │ "prompt_cached_tokens": 24271, + │ "prompt_tokens": 24610, + │ "tokens": 24670 │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { @@ -620,13 +637,16 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-sonnet-4-5-20250929" + │ │ "model": "claude-sonnet-4-5-20250929", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 3, + │ │ "completion_tokens": 259, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 24171, │ │ "prompt_tokens": 24181, - │ │ "tokens": 24184 + │ │ "tokens": 24440 │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -747,14 +767,16 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ │ metrics: { - │ │ │ "completion_tokens": 57, - │ │ │ "prompt_cache_creation_tokens": 1359, + │ │ │ "completion_tokens": 74, + │ │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ │ "prompt_cache_creation_5m_tokens": 1359, │ │ │ "prompt_cached_tokens": 4548, │ │ │ "prompt_tokens": 5910, - │ │ │ "tokens": 5967 + │ │ │ "tokens": 5984 │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { @@ -822,14 +844,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-sonnet-4-5-20250929" + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 308, - │ "prompt_cache_creation_tokens": 353, - │ "prompt_cached_tokens": 48342, - │ "prompt_tokens": 48703, - │ "tokens": 49011 + │ "completion_tokens": 109, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 353, + │ "prompt_cached_tokens": 24171, + │ "prompt_tokens": 24532, + │ "tokens": 24641 │ } └── claude-agent-failure-operation metadata: { @@ -891,14 +915,14 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 7, - │ "prompt_cache_creation_tokens": 2374, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 2374, │ "prompt_cached_tokens": 21870, - │ "prompt_tokens": 24254, - │ "tokens": 24261 + │ "prompt_tokens": 24254 │ } ├── tool: calculator/calculator [tool] │ input: { @@ -962,12 +986,12 @@ span_tree: } ] metadata: { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } metrics: { - "completion_tokens": 260, - "prompt_cache_creation_tokens": 2605, - "prompt_cached_tokens": 46114, - "prompt_tokens": 48727, - "tokens": 48987 + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 231, + "prompt_cached_tokens": 24244, + "prompt_tokens": 24483 } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json index 3ac00979c..33a985b81 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json @@ -52,13 +52,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 7, + "completion_tokens": 168, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 24243, "prompt_tokens": 24253, - "tokens": 24260 + "tokens": 24421 } }, { @@ -140,14 +143,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 209, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 48486, - "prompt_tokens": 48693, - "tokens": 48902 + "completion_tokens": 48, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 24243, + "prompt_tokens": 24450, + "tokens": 24498 } } ], @@ -217,10 +222,13 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { "completion_tokens": 475, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 24110, "prompt_tokens": 24120, "tokens": 24595 @@ -271,14 +279,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 4, - "prompt_cache_creation_tokens": 488, + "completion_tokens": 148, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 488, "prompt_cached_tokens": 24110, "prompt_tokens": 24608, - "tokens": 24612 + "tokens": 24756 } } ], @@ -369,13 +379,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, + "completion_tokens": 234, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 24271, "prompt_tokens": 24281, - "tokens": 24284 + "tokens": 24515 } }, { @@ -422,14 +435,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 2201, + "completion_tokens": 106, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2201, "prompt_cached_tokens": 16794, "prompt_tokens": 18998, - "tokens": 19001 + "tokens": 19104 } }, { @@ -600,14 +615,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 288, - "prompt_cache_creation_tokens": 331, - "prompt_cached_tokens": 48542, - "prompt_tokens": 48881, - "tokens": 49169 + "completion_tokens": 60, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 331, + "prompt_cached_tokens": 24271, + "prompt_tokens": 24610, + "tokens": 24670 } } ], @@ -685,13 +702,16 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, + "completion_tokens": 259, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 24171, "prompt_tokens": 24181, - "tokens": 24184 + "tokens": 24440 } }, { @@ -737,14 +757,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 57, - "prompt_cache_creation_tokens": 1359, + "completion_tokens": 74, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 1359, "prompt_cached_tokens": 4548, "prompt_tokens": 5910, - "tokens": 5967 + "tokens": 5984 } }, { @@ -909,14 +931,16 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 308, - "prompt_cache_creation_tokens": 353, - "prompt_cached_tokens": 48342, - "prompt_tokens": 48703, - "tokens": 49011 + "completion_tokens": 109, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 353, + "prompt_cached_tokens": 24171, + "prompt_tokens": 24532, + "tokens": 24641 } } ], @@ -995,14 +1019,14 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 7, - "prompt_cache_creation_tokens": 2374, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2374, "prompt_cached_tokens": 21870, - "prompt_tokens": 24254, - "tokens": 24261 + "prompt_tokens": 24254 } }, { @@ -1078,14 +1102,14 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 260, - "prompt_cache_creation_tokens": 2605, - "prompt_cached_tokens": 46114, - "prompt_tokens": 48727, - "tokens": 48987 + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 231, + "prompt_cached_tokens": 24244, + "prompt_tokens": 24483 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt index 4fd1913ec..0b6e46dea 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt @@ -64,13 +64,16 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 7, + │ │ "completion_tokens": 168, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 24243, │ │ "prompt_tokens": 24253, - │ │ "tokens": 24260 + │ │ "tokens": 24421 │ │ } │ ├── tool: calculator/calculator [tool] │ │ input: { @@ -140,14 +143,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 209, - │ "prompt_cache_creation_tokens": 199, - │ "prompt_cached_tokens": 48486, - │ "prompt_tokens": 48693, - │ "tokens": 48902 + │ "completion_tokens": 48, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 199, + │ "prompt_cached_tokens": 24243, + │ "prompt_tokens": 24450, + │ "tokens": 24498 │ } ├── claude-agent-async-prompt-operation │ metadata: { @@ -220,10 +225,13 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { │ │ "completion_tokens": 475, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 24110, │ │ "prompt_tokens": 24120, │ │ "tokens": 24595 @@ -270,14 +278,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 488, + │ "completion_tokens": 148, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 488, │ "prompt_cached_tokens": 24110, │ "prompt_tokens": 24608, - │ "tokens": 24612 + │ "tokens": 24756 │ } ├── claude-agent-subagent-operation │ metadata: { @@ -342,13 +352,16 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 3, + │ │ "completion_tokens": 234, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 24271, │ │ "prompt_tokens": 24281, - │ │ "tokens": 24284 + │ │ "tokens": 24515 │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -470,14 +483,16 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ │ metrics: { - │ │ │ "completion_tokens": 3, - │ │ │ "prompt_cache_creation_tokens": 2201, + │ │ │ "completion_tokens": 106, + │ │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ │ "prompt_cache_creation_5m_tokens": 2201, │ │ │ "prompt_cached_tokens": 16794, │ │ │ "prompt_tokens": 18998, - │ │ │ "tokens": 19001 + │ │ │ "tokens": 19104 │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { @@ -547,14 +562,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 288, - │ "prompt_cache_creation_tokens": 331, - │ "prompt_cached_tokens": 48542, - │ "prompt_tokens": 48881, - │ "tokens": 49169 + │ "completion_tokens": 60, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 331, + │ "prompt_cached_tokens": 24271, + │ "prompt_tokens": 24610, + │ "tokens": 24670 │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { @@ -620,13 +637,16 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-sonnet-4-5-20250929" + │ │ "model": "claude-sonnet-4-5-20250929", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 3, + │ │ "completion_tokens": 259, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 24171, │ │ "prompt_tokens": 24181, - │ │ "tokens": 24184 + │ │ "tokens": 24440 │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -747,14 +767,16 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ │ metrics: { - │ │ │ "completion_tokens": 57, - │ │ │ "prompt_cache_creation_tokens": 1359, + │ │ │ "completion_tokens": 74, + │ │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ │ "prompt_cache_creation_5m_tokens": 1359, │ │ │ "prompt_cached_tokens": 4548, │ │ │ "prompt_tokens": 5910, - │ │ │ "tokens": 5967 + │ │ │ "tokens": 5984 │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { @@ -822,14 +844,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-sonnet-4-5-20250929" + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 308, - │ "prompt_cache_creation_tokens": 353, - │ "prompt_cached_tokens": 48342, - │ "prompt_tokens": 48703, - │ "tokens": 49011 + │ "completion_tokens": 109, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 353, + │ "prompt_cached_tokens": 24171, + │ "prompt_tokens": 24532, + │ "tokens": 24641 │ } └── claude-agent-failure-operation metadata: { @@ -891,14 +915,14 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 7, - │ "prompt_cache_creation_tokens": 2374, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 2374, │ "prompt_cached_tokens": 21870, - │ "prompt_tokens": 24254, - │ "tokens": 24261 + │ "prompt_tokens": 24254 │ } ├── tool: calculator/calculator [tool] │ input: { @@ -962,12 +986,12 @@ span_tree: } ] metadata: { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } metrics: { - "completion_tokens": 260, - "prompt_cache_creation_tokens": 2605, - "prompt_cached_tokens": 46114, - "prompt_tokens": 48727, - "tokens": 48987 + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 231, + "prompt_cached_tokens": 24244, + "prompt_tokens": 24483 } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json index d263e9f02..1921e7459 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json @@ -52,13 +52,15 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 18783, + "completion_tokens": 174, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18783, "prompt_tokens": 18793, - "tokens": 18796 + "tokens": 18967 } }, { @@ -140,14 +142,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 216, - "prompt_cache_creation_tokens": 18988, + "completion_tokens": 45, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 205, "prompt_cached_tokens": 18783, - "prompt_tokens": 37779, - "tokens": 37995 + "prompt_tokens": 18996, + "tokens": 19041 } } ], @@ -217,11 +221,13 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { "completion_tokens": 171, - "prompt_cache_creation_tokens": 18650, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18650, "prompt_tokens": 18660, "tokens": 18831 } @@ -271,14 +277,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 19, - "prompt_cache_creation_tokens": 184, + "completion_tokens": 190, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 184, "prompt_cached_tokens": 18650, "prompt_tokens": 18844, - "tokens": 18863 + "tokens": 19034 } } ], @@ -369,13 +377,15 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 18811, + "completion_tokens": 215, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18811, "prompt_tokens": 18821, - "tokens": 18824 + "tokens": 19036 } }, { @@ -422,13 +432,15 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 26759, + "completion_tokens": 106, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 26759, "prompt_tokens": 26762, - "tokens": 26765 + "tokens": 26868 } }, { @@ -577,14 +589,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 291, - "prompt_cache_creation_tokens": 19118, + "completion_tokens": 82, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 307, "prompt_cached_tokens": 18811, - "prompt_tokens": 37937, - "tokens": 38228 + "prompt_tokens": 19126, + "tokens": 19208 } } ], @@ -662,13 +676,15 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 4, - "prompt_cache_creation_tokens": 18711, + "completion_tokens": 169, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18711, "prompt_tokens": 18721, - "tokens": 18725 + "tokens": 18890 } }, { @@ -714,13 +730,15 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 50, - "prompt_cache_creation_tokens": 17229, + "completion_tokens": 76, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 17229, "prompt_tokens": 17232, - "tokens": 17282 + "tokens": 17308 } }, { @@ -863,14 +881,16 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 281, - "prompt_cache_creation_tokens": 18961, + "completion_tokens": 166, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 250, "prompt_cached_tokens": 18711, - "prompt_tokens": 37680, - "tokens": 37961 + "prompt_tokens": 18969, + "tokens": 19135 } } ], @@ -949,13 +969,14 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 8, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 18784, - "prompt_tokens": 18794, - "tokens": 18802 + "prompt_tokens": 18794 } }, { @@ -1031,14 +1052,14 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 225, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 37568, - "prompt_tokens": 37775, - "tokens": 38000 + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 18784, + "prompt_tokens": 18991 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt index f2d056531..aae48e805 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt @@ -64,13 +64,15 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cache_creation_tokens": 18783, + │ │ "completion_tokens": 174, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 18783, │ │ "prompt_tokens": 18793, - │ │ "tokens": 18796 + │ │ "tokens": 18967 │ │ } │ ├── tool: calculator/calculator [tool] │ │ input: { @@ -140,14 +142,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 216, - │ "prompt_cache_creation_tokens": 18988, + │ "completion_tokens": 45, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 205, │ "prompt_cached_tokens": 18783, - │ "prompt_tokens": 37779, - │ "tokens": 37995 + │ "prompt_tokens": 18996, + │ "tokens": 19041 │ } ├── claude-agent-async-prompt-operation │ metadata: { @@ -220,11 +224,13 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { │ │ "completion_tokens": 171, - │ │ "prompt_cache_creation_tokens": 18650, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 18650, │ │ "prompt_tokens": 18660, │ │ "tokens": 18831 │ │ } @@ -270,14 +276,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 19, - │ "prompt_cache_creation_tokens": 184, + │ "completion_tokens": 190, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 184, │ "prompt_cached_tokens": 18650, │ "prompt_tokens": 18844, - │ "tokens": 18863 + │ "tokens": 19034 │ } ├── claude-agent-subagent-operation │ metadata: { @@ -342,13 +350,15 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cache_creation_tokens": 18811, + │ │ "completion_tokens": 215, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 18811, │ │ "prompt_tokens": 18821, - │ │ "tokens": 18824 + │ │ "tokens": 19036 │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -448,13 +458,15 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ │ metrics: { - │ │ │ "completion_tokens": 3, - │ │ │ "prompt_cache_creation_tokens": 26759, + │ │ │ "completion_tokens": 106, + │ │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ │ "prompt_cache_creation_5m_tokens": 26759, │ │ │ "prompt_tokens": 26762, - │ │ │ "tokens": 26765 + │ │ │ "tokens": 26868 │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { @@ -524,14 +536,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 291, - │ "prompt_cache_creation_tokens": 19118, + │ "completion_tokens": 82, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 307, │ "prompt_cached_tokens": 18811, - │ "prompt_tokens": 37937, - │ "tokens": 38228 + │ "prompt_tokens": 19126, + │ "tokens": 19208 │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { @@ -597,13 +611,15 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-sonnet-4-5-20250929" + │ │ "model": "claude-sonnet-4-5-20250929", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 4, - │ │ "prompt_cache_creation_tokens": 18711, + │ │ "completion_tokens": 169, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 18711, │ │ "prompt_tokens": 18721, - │ │ "tokens": 18725 + │ │ "tokens": 18890 │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -702,13 +718,15 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ │ metrics: { - │ │ │ "completion_tokens": 50, - │ │ │ "prompt_cache_creation_tokens": 17229, + │ │ │ "completion_tokens": 76, + │ │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ │ "prompt_cache_creation_5m_tokens": 17229, │ │ │ "prompt_tokens": 17232, - │ │ │ "tokens": 17282 + │ │ │ "tokens": 17308 │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { @@ -776,14 +794,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-sonnet-4-5-20250929" + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 281, - │ "prompt_cache_creation_tokens": 18961, + │ "completion_tokens": 166, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 250, │ "prompt_cached_tokens": 18711, - │ "prompt_tokens": 37680, - │ "tokens": 37961 + │ "prompt_tokens": 18969, + │ "tokens": 19135 │ } └── claude-agent-failure-operation metadata: { @@ -845,13 +865,14 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 8, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 18784, - │ "prompt_tokens": 18794, - │ "tokens": 18802 + │ "prompt_tokens": 18794 │ } ├── tool: calculator/calculator [tool] │ input: { @@ -915,12 +936,12 @@ span_tree: } ] metadata: { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } metrics: { - "completion_tokens": 225, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 37568, - "prompt_tokens": 37775, - "tokens": 38000 + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 18784, + "prompt_tokens": 18991 } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts b/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts index c7983324d..23f90820c 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts @@ -21,6 +21,21 @@ import { import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; +type ExpectedUsage = { + message_id?: string; + parent_tool_use_id?: string | null; + usage?: { + cache_creation_input_tokens?: number; + cache_creation?: { + ephemeral_5m_input_tokens?: number; + ephemeral_1h_input_tokens?: number; + }; + cache_read_input_tokens?: number; + input_tokens?: number; + output_tokens?: number; + }; +}; + type RunClaudeAgentSDKScenario = (harness: { runNodeScenarioDir: (options: { entry: string; @@ -59,11 +74,6 @@ const SNAPSHOT_METADATA_KEYS = [ "claude_agent_sdk.tool_use_count", "claude_agent_sdk.total_tokens", ] as const; -const OMITTED_METRIC_KEYS = new Set([ - "prompt_cached_tokens", - "prompt_cache_creation_tokens", -]); - function summarizeSpan( event: CapturedLogEvent | undefined, overrides?: { @@ -79,12 +89,7 @@ function summarizeSpan( const summary = summarizeWrapperContract(event, [ ...SNAPSHOT_METADATA_KEYS, ]) as Record; - const metricKeys = Array.isArray(summary.metric_keys) - ? summary.metric_keys.filter( - (key): key is string => - typeof key === "string" && !OMITTED_METRIC_KEYS.has(key), - ) - : summary.metric_keys; + const metricKeys = summary.metric_keys; const input = event.input as | Array<{ content?: string; message?: { content?: string } }> | undefined; @@ -145,6 +150,99 @@ function summarizeSpan( return summary; } +function metricsFromTranscriptUsage(expected: ExpectedUsage): { + completion_tokens: number | undefined; + prompt_cache_creation_tokens: number | undefined; + prompt_cache_creation_5m_tokens: number | undefined; + prompt_cache_creation_1h_tokens: number | undefined; + prompt_cached_tokens: number; + prompt_tokens: number; + tokens: number; +} { + const inputTokens = expected.usage?.input_tokens ?? 0; + const cachedTokens = expected.usage?.cache_read_input_tokens ?? 0; + const aggregateCacheCreationTokens = + expected.usage?.cache_creation_input_tokens ?? 0; + const cacheCreation5mTokens = + expected.usage?.cache_creation?.ephemeral_5m_input_tokens; + const cacheCreation1hTokens = + expected.usage?.cache_creation?.ephemeral_1h_input_tokens; + const hasCacheCreationBreakdown = + cacheCreation5mTokens !== undefined || cacheCreation1hTokens !== undefined; + const effectiveCacheCreationTokens = hasCacheCreationBreakdown + ? (cacheCreation5mTokens ?? 0) + (cacheCreation1hTokens ?? 0) + : aggregateCacheCreationTokens; + const completionTokens = expected.usage?.output_tokens; + const promptTokens = + inputTokens + cachedTokens + effectiveCacheCreationTokens; + + return { + completion_tokens: completionTokens, + prompt_cache_creation_tokens: + !hasCacheCreationBreakdown && aggregateCacheCreationTokens > 0 + ? aggregateCacheCreationTokens + : undefined, + prompt_cache_creation_5m_tokens: cacheCreation5mTokens, + prompt_cache_creation_1h_tokens: cacheCreation1hTokens, + prompt_cached_tokens: cachedTokens, + prompt_tokens: promptTokens, + tokens: promptTokens + (completionTokens ?? 0), + }; +} + +function spanUsageMatchesTranscript( + span: CapturedLogEvent | undefined, + expected: ExpectedUsage, +): boolean { + const expectedMetrics = metricsFromTranscriptUsage(expected); + return ( + span?.metrics?.prompt_tokens === expectedMetrics.prompt_tokens && + span.metrics?.completion_tokens === expectedMetrics.completion_tokens && + span.metrics?.tokens === expectedMetrics.tokens && + (span.metrics?.prompt_cached_tokens ?? 0) === + expectedMetrics.prompt_cached_tokens && + span.metrics?.prompt_cache_creation_tokens === + expectedMetrics.prompt_cache_creation_tokens && + span.metrics?.prompt_cache_creation_5m_tokens === + expectedMetrics.prompt_cache_creation_5m_tokens && + span.metrics?.prompt_cache_creation_1h_tokens === + expectedMetrics.prompt_cache_creation_1h_tokens + ); +} + +function expectSpanUsageToMatchTranscript( + span: CapturedLogEvent | undefined, + expected: ExpectedUsage, +): void { + const expectedMetrics = metricsFromTranscriptUsage(expected); + expect(span?.metrics).toMatchObject({ + ...(expectedMetrics.prompt_cache_creation_tokens !== undefined && { + prompt_cache_creation_tokens: + expectedMetrics.prompt_cache_creation_tokens, + }), + ...(expectedMetrics.prompt_cache_creation_5m_tokens !== undefined && { + prompt_cache_creation_5m_tokens: + expectedMetrics.prompt_cache_creation_5m_tokens, + }), + ...(expectedMetrics.prompt_cache_creation_1h_tokens !== undefined && { + prompt_cache_creation_1h_tokens: + expectedMetrics.prompt_cache_creation_1h_tokens, + }), + ...(expectedMetrics.prompt_cached_tokens > 0 && { + prompt_cached_tokens: expectedMetrics.prompt_cached_tokens, + }), + completion_tokens: expectedMetrics.completion_tokens, + prompt_tokens: expectedMetrics.prompt_tokens, + tokens: expectedMetrics.tokens, + }); + if ( + expectedMetrics.prompt_cache_creation_5m_tokens !== undefined || + expectedMetrics.prompt_cache_creation_1h_tokens !== undefined + ) { + expect(span?.metrics?.prompt_cache_creation_tokens).toBeUndefined(); + } +} + function findToolSpanByOperation( events: CapturedLogEvent[], operation: "add" | "divide" | "multiply" | "subtract", @@ -483,6 +581,42 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); }); + test( + "attributes exact provider stream usage to each basic llm call", + testConfig, + () => { + const operation = findLatestSpan( + events, + "claude-agent-basic-operation", + ); + const task = findChildSpans( + events, + "Claude Agent", + operation?.span.id, + ).at(-1); + const expectedUsageSpan = findChildSpans( + events, + "claude-agent-basic-partial-usage", + operation?.span.id, + ).at(-1); + const expectedUsage = expectedUsageSpan?.output as + | ExpectedUsage[] + | undefined; + const llmSpans = findChildSpans( + events, + "anthropic.messages.create", + task?.span.id, + ); + + expect(expectedUsage?.length).toBeGreaterThan(1); + expect(llmSpans).toHaveLength(expectedUsage?.length ?? 0); + + for (const [index, expected] of (expectedUsage ?? []).entries()) { + expectSpanUsageToMatchTranscript(llmSpans[index], expected); + } + }, + ); + if (options.assertLocalToolHandlerParenting) { test( "nests local tool handler spans under tool spans", @@ -558,6 +692,112 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { }, ); + test( + "recovers exact usage when partial messages are disabled", + testConfig, + () => { + const operation = findLatestSpan( + events, + "claude-agent-async-prompt-operation", + ); + const task = findChildSpans( + events, + "Claude Agent", + operation?.span.id, + ).at(-1); + const expectedUsageSpan = findChildSpans( + events, + "claude-agent-async-prompt-transcript-usage", + operation?.span.id, + ).at(-1); + const expectedUsage = expectedUsageSpan?.output as + | ExpectedUsage[] + | undefined; + const llmSpans = findChildSpans( + events, + "anthropic.messages.create", + task?.span.id, + ); + + expect(expectedUsage?.length).toBeGreaterThan(0); + expect(llmSpans).toHaveLength(expectedUsage?.length ?? 0); + for (const [index, expected] of (expectedUsage ?? []).entries()) { + expectSpanUsageToMatchTranscript(llmSpans[index], expected); + } + }, + ); + + test( + "recovers root and separate subagent transcript usage", + testConfig, + () => { + const operation = findLatestSpan( + events, + "claude-agent-subagent-operation", + ); + const taskRoot = findOperationTaskRoot( + events, + "claude-agent-subagent-operation", + ); + const nestedTask = findSubAgentTaskSpan(events, taskRoot?.span.id); + const expectedUsageSpan = findChildSpans( + events, + "claude-agent-subagent-transcript-usage", + operation?.span.id, + ).at(-1); + const expectedUsage = (expectedUsageSpan?.output ?? + []) as ExpectedUsage[]; + const expectedRootUsage = expectedUsage.filter( + (entry) => entry.parent_tool_use_id === null, + ); + const expectedSubagentUsage = expectedUsage.filter( + (entry) => typeof entry.parent_tool_use_id === "string", + ); + const rootLlmSpans = findChildSpans( + events, + "anthropic.messages.create", + taskRoot?.span.id, + ); + const subagentLlmSpans = findChildSpans( + events, + "anthropic.messages.create", + nestedTask?.span.id, + ); + + expect(expectedRootUsage.length).toBeGreaterThan(0); + expect(expectedSubagentUsage.length).toBeGreaterThan(0); + expect(rootLlmSpans).toHaveLength(expectedRootUsage.length); + expect(subagentLlmSpans.length).toBeGreaterThan(0); + for (const [index, expected] of expectedRootUsage.entries()) { + expectSpanUsageToMatchTranscript(rootLlmSpans[index], expected); + } + for (const span of subagentLlmSpans) { + const hasMatch = expectedSubagentUsage.some((expected) => + spanUsageMatchesTranscript(span, expected), + ); + if (!hasMatch) { + throw new Error( + `Subagent span usage did not match transcript: ${JSON.stringify({ expectedSubagentUsage, metrics: span.metrics })}`, + ); + } + } + }, + ); + + test( + "does not store aggregate token usage on task spans", + testConfig, + () => { + const taskSpans = findAllSpans(events, "Claude Agent"); + expect(taskSpans.length).toBeGreaterThan(0); + for (const task of taskSpans) { + expect(task.metrics?.prompt_tokens).toBeUndefined(); + expect(task.metrics?.completion_tokens).toBeUndefined(); + expect(task.metrics?.tokens).toBeUndefined(); + } + }, + ); + test("captures nested subagent task hierarchy", testConfig, () => { const operation = findLatestSpan( events, @@ -719,6 +959,34 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { ); } + test( + "falls back to prompt usage when the transcript is unavailable", + testConfig, + () => { + const operation = findLatestSpan( + events, + "claude-agent-failure-operation", + ); + const task = findChildSpans( + events, + "Claude Agent", + operation?.span.id, + ).at(-1); + const llmSpans = findChildSpans( + events, + "anthropic.messages.create", + task?.span.id, + ); + + expect(llmSpans.length).toBeGreaterThan(0); + for (const llm of llmSpans) { + expect(llm.metrics?.prompt_tokens).toEqual(expect.any(Number)); + expect(llm.metrics?.completion_tokens).toBeUndefined(); + expect(llm.metrics?.tokens).toBeUndefined(); + } + }, + ); + test("captures tool failure details", testConfig, () => { const operation = findLatestSpan( events, @@ -753,9 +1021,19 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { "matches the shared span tree snapshot", testConfig, async ({ expect }) => { - await matchSpanTreeSnapshot(events, snapshotPath, { - snapshotExpect: expect, - }); + await matchSpanTreeSnapshot( + events.filter( + (event) => + event.span.name !== "claude-agent-basic-partial-usage" && + event.span.name !== + "claude-agent-async-prompt-transcript-usage" && + event.span.name !== "claude-agent-subagent-transcript-usage", + ), + snapshotPath, + { + snapshotExpect: expect, + }, + ); }, ); }); diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/cassette-filter.mjs b/e2e/scenarios/claude-agent-sdk-instrumentation/cassette-filter.mjs index 7890b9d5c..2bab107db 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/cassette-filter.mjs +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/cassette-filter.mjs @@ -5,10 +5,24 @@ export const filter = [ { normalizeRequest(req) { const url = new URL(req.url); - if (req.method === "POST" && url.hostname === "api.anthropic.com") { - return { ...req, body: { kind: "empty" } }; + if (req.method !== "POST" || url.hostname !== "api.anthropic.com") { + return req; } - return req; + + // Claude can send a title request at the same time as a model request. + // Give each request class a separate cassette sequence. Large request + // bodies contain volatile Claude system data, so match them by order. + const requestClass = + req.body?.kind === "json" && req.body.value?.output_config + ? "title" + : "model"; + url.pathname = `${url.pathname}/__cassette_${requestClass}`; + + return { + ...req, + body: { kind: "empty" }, + url: url.toString(), + }; }, }, ]; diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs b/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs index 4d565d2a0..63a0e82a4 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs @@ -1,4 +1,5 @@ -import { traced, wrapClaudeAgentSDK } from "braintrust"; +import { readFile } from "node:fs/promises"; +import { startSpan, traced, wrapClaudeAgentSDK } from "braintrust"; import { collectAsync, runOperation, @@ -22,6 +23,253 @@ function makePromptMessage(content) { }; } +function collectFinalPartialUsage(messages) { + const activeMessageByParent = new Map(); + const usageByMessageId = new Map(); + + for (const message of messages) { + if (message.type !== "stream_event") { + continue; + } + + const parentKey = message.parent_tool_use_id ?? "__root__"; + const event = message.event; + if (event?.type === "message_start" && event.message?.id) { + activeMessageByParent.set(parentKey, event.message.id); + usageByMessageId.set(event.message.id, { + message_id: event.message.id, + parent_tool_use_id: message.parent_tool_use_id ?? null, + usage: { ...event.message.usage }, + }); + continue; + } + + const messageId = activeMessageByParent.get(parentKey); + if (!messageId) { + continue; + } + + if (event?.type === "message_delta") { + const captured = usageByMessageId.get(messageId); + if (captured && event.usage) { + Object.assign(captured.usage, event.usage); + } + } else if (event?.type === "message_stop") { + activeMessageByParent.delete(parentKey); + } + } + + return [...usageByMessageId.values()]; +} + +function assertNoPartialMessages(messages) { + if (messages.some((message) => message.type === "stream_event")) { + throw new Error( + "Braintrust exposed internally enabled Claude Agent SDK partial messages", + ); + } +} + +function createTranscriptCapture() { + const state = { + rootPath: undefined, + subagentPathByToolUseId: new Map(), + }; + + const captureRootTranscriptPath = async (input) => { + if ( + (input.hook_event_name === "SessionStart" || + input.hook_event_name === "SessionEnd" || + input.hook_event_name === "UserPromptSubmit") && + typeof input.transcript_path === "string" + ) { + state.rootPath = input.transcript_path; + } + return {}; + }; + + return { + hooks: { + SessionStart: [{ hooks: [captureRootTranscriptPath] }], + SessionEnd: [{ hooks: [captureRootTranscriptPath] }], + UserPromptSubmit: [{ hooks: [captureRootTranscriptPath] }], + SubagentStop: [ + { + hooks: [ + async (input, toolUseId) => { + if ( + input.hook_event_name === "SubagentStop" && + typeof toolUseId === "string" && + typeof input.agent_transcript_path === "string" + ) { + state.subagentPathByToolUseId.set( + toolUseId, + input.agent_transcript_path, + ); + } + return {}; + }, + ], + }, + ], + }, + state, + }; +} + +function validTokenCount(value) { + return typeof value === "number" && + Number.isFinite(value) && + Number.isInteger(value) && + value >= 0 + ? value + : undefined; +} + +async function readFinalTranscriptUsage(transcriptPath, parentToolUseId) { + const text = await readFile(transcriptPath, "utf8"); + const usageByMessageId = new Map(); + + for (const line of text.split("\n")) { + if (!line.trim()) { + continue; + } + + let row; + try { + row = JSON.parse(line); + } catch { + continue; + } + + const messageId = row?.type === "assistant" ? row.message?.id : undefined; + const usage = row?.message?.usage; + if (typeof messageId !== "string" || !usage) { + continue; + } + + const inputTokens = validTokenCount(usage.input_tokens); + const outputTokens = validTokenCount(usage.output_tokens); + const cacheReadTokens = validTokenCount(usage.cache_read_input_tokens ?? 0); + const cacheCreationTokens = validTokenCount( + usage.cache_creation_input_tokens ?? 0, + ); + let cacheCreation; + if (usage.cache_creation !== undefined) { + if (!usage.cache_creation || typeof usage.cache_creation !== "object") { + continue; + } + + const cacheCreation5mTokens = validTokenCount( + usage.cache_creation.ephemeral_5m_input_tokens, + ); + const cacheCreation1hTokens = validTokenCount( + usage.cache_creation.ephemeral_1h_input_tokens, + ); + if ( + (usage.cache_creation.ephemeral_5m_input_tokens !== undefined && + cacheCreation5mTokens === undefined) || + (usage.cache_creation.ephemeral_1h_input_tokens !== undefined && + cacheCreation1hTokens === undefined) + ) { + continue; + } + if ( + cacheCreation5mTokens !== undefined || + cacheCreation1hTokens !== undefined + ) { + cacheCreation = { + ...(cacheCreation5mTokens !== undefined && { + ephemeral_5m_input_tokens: cacheCreation5mTokens, + }), + ...(cacheCreation1hTokens !== undefined && { + ephemeral_1h_input_tokens: cacheCreation1hTokens, + }), + }; + } + } + if ( + inputTokens === undefined || + outputTokens === undefined || + cacheReadTokens === undefined || + cacheCreationTokens === undefined + ) { + continue; + } + + usageByMessageId.set(messageId, { + message_id: messageId, + parent_tool_use_id: parentToolUseId, + usage: { + cache_creation_input_tokens: cacheCreationTokens, + cache_read_input_tokens: cacheReadTokens, + input_tokens: inputTokens, + output_tokens: outputTokens, + ...(cacheCreation && { cache_creation: cacheCreation }), + }, + }); + } + + return [...usageByMessageId.values()]; +} + +async function collectCapturedTranscriptUsage(capture, messages) { + if (!capture.state.rootPath) { + throw new Error("User session hook did not receive a transcript path"); + } + + const parentToolUseIdByMessageId = new Map( + messages + .filter( + (message) => + message.type === "assistant" && + typeof message.message?.id === "string", + ) + .map((message) => [ + message.message.id, + message.parent_tool_use_id ?? null, + ]), + ); + const usage = await readFinalTranscriptUsage(capture.state.rootPath, null); + for (const transcriptPath of capture.state.subagentPathByToolUseId.values()) { + const subagentUsage = await readFinalTranscriptUsage(transcriptPath, null); + for (const entry of subagentUsage) { + if (parentToolUseIdByMessageId.has(entry.message_id)) { + entry.parent_tool_use_id = parentToolUseIdByMessageId.get( + entry.message_id, + ); + usage.push(entry); + } + } + } + return usage; +} + +async function logExpectedTranscriptUsage(name, capture, messages) { + const expectedUsageSpan = startSpan({ name }); + expectedUsageSpan.log({ + output: await collectCapturedTranscriptUsage(capture, messages), + }); + expectedUsageSpan.end(); +} + +async function collectAsyncAndAssertMessagesUnchanged(records) { + const messages = []; + const originalMessages = []; + for await (const message of records) { + messages.push(message); + originalMessages.push(JSON.stringify(message)); + } + + for (const [index, message] of messages.entries()) { + if (JSON.stringify(message) !== originalMessages[index]) { + throw new Error("Braintrust mutated a Claude Agent SDK message"); + } + } + + return messages; +} + async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { const instrumentedSDK = decorateSDK ? decorateSDK(sdk) : sdk; const { createSdkMcpServer, query, tool } = instrumentedSDK; @@ -76,11 +324,12 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { await runTracedScenario({ callback: async () => { await runOperation("claude-agent-basic-operation", "basic", async () => { - await collectAsync( + const messages = await collectAsyncAndAssertMessagesUnchanged( query({ prompt: "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", options: { + includePartialMessages: true, mcpServers: { calculator: calculatorServer, }, @@ -89,25 +338,41 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { }, }), ); + const expectedUsageSpan = startSpan({ + name: "claude-agent-basic-partial-usage", + }); + expectedUsageSpan.log({ + output: collectFinalPartialUsage(messages), + }); + expectedUsageSpan.end(); }); await runOperation( "claude-agent-async-prompt-operation", "async-prompt", async () => { - await collectAsync( + const transcriptCapture = createTranscriptCapture(); + const messages = await collectAsyncAndAssertMessagesUnchanged( query({ prompt: (async function* () { yield makePromptMessage("Part 1"); yield makePromptMessage("Part 2"); })(), options: { + hooks: transcriptCapture.hooks, + includePartialMessages: false, maxTurns: 1, model: CLAUDE_AGENT_MODEL, permissionMode: "bypassPermissions", }, }), ); + assertNoPartialMessages(messages); + await logExpectedTranscriptUsage( + "claude-agent-async-prompt-transcript-usage", + transcriptCapture, + messages, + ); }, ); @@ -115,7 +380,8 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { "claude-agent-subagent-operation", "subagent", async () => { - await collectAsync( + const transcriptCapture = createTranscriptCapture(); + const messages = await collectAsyncAndAssertMessagesUnchanged( query({ prompt: "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", @@ -129,6 +395,7 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { }, }, allowedTools: ["Task"], + hooks: transcriptCapture.hooks, mcpServers: { calculator: calculatorServer, }, @@ -137,6 +404,12 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { }, }), ); + assertNoPartialMessages(messages); + await logExpectedTranscriptUsage( + "claude-agent-subagent-transcript-usage", + transcriptCapture, + messages, + ); }, ); @@ -171,7 +444,7 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { "claude-agent-failure-operation", "failure", async () => { - await collectAsync( + const messages = await collectAsyncAndAssertMessagesUnchanged( query({ prompt: "Use the calculator tool to divide 2 by 0. Do not recover from the error.", @@ -181,9 +454,11 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { }, model: CLAUDE_AGENT_MODEL, permissionMode: "bypassPermissions", + persistSession: false, }, }), ); + assertNoPartialMessages(messages); }, ); }, diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index c8aa0b955..f21f65c81 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -111,6 +111,54 @@ describe("parseMetricsFromUsage", () => { }); }); + it("should surface the per-TTL cache creation breakdown", () => { + const usage = { + input_tokens: 100, + output_tokens: 50, + cache_read_input_tokens: 25, + cache_creation_input_tokens: 30, + cache_creation: { + ephemeral_1h_input_tokens: 10, + ephemeral_5m_input_tokens: 20, + }, + }; + + const result = parseMetricsFromUsageForTest(usage); + + // The aggregate is still parsed here; `finalizeAnthropicTokens` is what + // reduces the span to the per-TTL representation. + expect(result).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_cached_tokens: 25, + prompt_cache_creation_tokens: 30, + prompt_cache_creation_1h_tokens: 10, + prompt_cache_creation_5m_tokens: 20, + }); + }); + + it("should ignore a null or partial cache creation breakdown", () => { + expect( + parseMetricsFromUsageForTest({ + input_tokens: 100, + output_tokens: 50, + cache_creation: null, + }), + ).toEqual({ prompt_tokens: 100, completion_tokens: 50 }); + + expect( + parseMetricsFromUsageForTest({ + input_tokens: 100, + output_tokens: 50, + cache_creation: { ephemeral_5m_input_tokens: 20 }, + }), + ).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_cache_creation_5m_tokens: 20, + }); + }); + it("should ignore non-number token values", () => { const usage = { input_tokens: "not a number", diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index 261223e91..a8dfea9b3 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -825,6 +825,24 @@ export function parseMetricsFromUsage( saveIfExistsTo("cache_read_input_tokens", "prompt_cached_tokens"); saveIfExistsTo("cache_creation_input_tokens", "prompt_cache_creation_tokens"); + // The 5m and 1h cache-write tiers are billed at different rates, so surface + // the per-TTL breakdown whenever the response carries it. It is an + // alternative representation of `cache_creation_input_tokens` rather than + // additional tokens; `finalizeAnthropicTokens` drops the aggregate so the + // span keeps a single representation. + if (isObject(usage.cache_creation)) { + const cacheCreation = usage.cache_creation; + for (const [source, target] of [ + ["ephemeral_5m_input_tokens", "prompt_cache_creation_5m_tokens"], + ["ephemeral_1h_input_tokens", "prompt_cache_creation_1h_tokens"], + ] as const) { + const value = cacheCreation[source]; + if (typeof value === "number") { + metrics[target] = value; + } + } + } + if (isObject(usage.server_tool_use)) { for (const [name, value] of Object.entries(usage.server_tool_use)) { if (typeof value === "number") { diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts index 6eef8c572..10c656014 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts +++ b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts @@ -1,12 +1,45 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; // Mock iso's newTracingChannel - must be before any imports that use it +const streamPatcherMock = vi.hoisted(() => ({ + options: undefined as + | { + onChunk?: (chunk: unknown) => void | Promise; + onComplete: () => void | Promise; + } + | undefined, +})); + vi.mock("../../isomorph", () => ({ default: { newTracingChannel: vi.fn(), + readFile: vi.fn(), }, })); +vi.mock("../../debug-logger", () => ({ + debugLogger: { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + }, +})); + +vi.mock("../core/stream-patcher", () => ({ + isAsyncIterable: vi.fn( + (val: unknown) => + val !== null && + typeof val === "object" && + Symbol.asyncIterator in val && + typeof (val as any)[Symbol.asyncIterator] === "function", + ), + patchStreamIfNeeded: vi.fn((stream, options) => { + streamPatcherMock.options = options; + return stream; + }), +})); + import { ClaudeAgentSDKPlugin } from "./claude-agent-sdk-plugin"; import iso from "../../isomorph"; import { startSpan } from "../../logger"; @@ -45,16 +78,9 @@ vi.mock("../../wrappers/attachment-utils", () => ({ processInputAttachments: vi.fn((input) => input), })); -vi.mock("../../wrappers/anthropic-tokens-util", () => ({ - extractAnthropicCacheTokens: vi.fn((read, creation) => ({ - prompt_cache_read_tokens: read, - prompt_cache_creation_tokens: creation, - })), - finalizeAnthropicTokens: vi.fn((metrics) => ({ - ...metrics, - tokens: (metrics.prompt_tokens || 0) + (metrics.completion_tokens || 0), - })), -})); +// `anthropic-tokens-util` is pure and owns the cache-creation representation +// rules, so these tests run the real implementation rather than a stand-in that +// could drift from it. vi.mock("../core", async (importOriginal) => { const actual = await importOriginal(); @@ -109,6 +135,7 @@ describe("ClaudeAgentSDKPlugin", () => { let mockUnsubscribe: any; beforeEach(() => { + streamPatcherMock.options = undefined; mockUnsubscribe = vi.fn(); mockChannel = { subscribe: vi.fn(), @@ -338,6 +365,418 @@ describe("ClaudeAgentSDKPlugin", () => { // Should not throw expect(true).toBe(true); }); + + it("recovers final usage from the transcript without changing query behavior", async () => { + const userSessionStart = vi.fn(async (..._args: unknown[]) => ({})); + const userSessionStartMatcher = { hooks: [userSessionStart] }; + const startEvent = { + arguments: [ + { + prompt: "Test", + options: { + hooks: { SessionStart: [userSessionStartMatcher] }, + includePartialMessages: false, + model: "claude-3-5-sonnet-20241022", + }, + }, + ], + }; + handlers.start(startEvent); + + const options = startEvent.arguments[0].options as any; + expect(options.includePartialMessages).toBe(false); + expect(options.hooks.SessionStart[0]).toBe(userSessionStartMatcher); + expect(options.hooks.SessionStart).toHaveLength(2); + expect(options.hooks.SessionEnd).toHaveLength(1); + expect(options.hooks.UserPromptSubmit).toHaveLength(1); + + const sessionStartInput = { + cwd: "/tmp", + hook_event_name: "SessionStart", + session_id: "session_1", + transcript_path: "/tmp/session.jsonl", + }; + await options.hooks.SessionStart[0].hooks[0]( + sessionStartInput, + undefined, + { signal: new AbortController().signal }, + ); + await options.hooks.SessionStart[1].hooks[0]( + sessionStartInput, + undefined, + { signal: new AbortController().signal }, + ); + expect(userSessionStart).toHaveBeenCalledOnce(); + + vi.mocked(iso.readFile!).mockResolvedValue( + new TextEncoder().encode( + [ + "not json", + JSON.stringify({ + type: "assistant", + message: { + id: "msg_1", + usage: { + cache_creation: { + ephemeral_1h_input_tokens: 0, + ephemeral_5m_input_tokens: 3, + }, + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 1, + }, + }, + }), + JSON.stringify({ + type: "assistant", + message: { + id: "msg_1", + usage: { + cache_creation: { + ephemeral_1h_input_tokens: 0, + ephemeral_5m_input_tokens: 3, + }, + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 40, + }, + }, + }), + JSON.stringify({ + type: "assistant", + message: { + id: "msg_1", + usage: { input_tokens: 10, output_tokens: -1 }, + }, + }), + ].join("\n"), + ), + ); + + const stream = { + async *[Symbol.asyncIterator]() { + // The patcher is mocked; messages are delivered below. + }, + }; + handlers.end(Object.assign(startEvent, { result: stream })); + + const assistantMessage = { + type: "assistant", + message: { + content: [{ text: "Response", type: "text" }], + id: "msg_1", + model: "claude-3-5-sonnet-20241022", + role: "assistant", + usage: { + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 1, + }, + }, + parent_tool_use_id: null, + }; + const originalAssistantMessage = JSON.stringify(assistantMessage); + await streamPatcherMock.options?.onChunk?.(assistantMessage); + await streamPatcherMock.options?.onChunk?.({ + num_turns: 1, + session_id: "session_1", + type: "result", + usage: { + cache_creation_input_tokens: 999, + cache_read_input_tokens: 999, + input_tokens: 999, + output_tokens: 999, + }, + }); + await streamPatcherMock.options?.onComplete(); + + expect(JSON.stringify(assistantMessage)).toBe(originalAssistantMessage); + expect(iso.readFile).toHaveBeenCalledWith("/tmp/session.jsonl"); + + const llmSpanCallIndex = vi + .mocked(startSpan) + .mock.calls.findIndex( + ([args]) => + args && + typeof args === "object" && + "name" in args && + args.name === "anthropic.messages.create", + ); + expect(llmSpanCallIndex).toBeGreaterThan(-1); + const llmSpan = + vi.mocked(startSpan).mock.results[llmSpanCallIndex]?.value; + expect(llmSpan?.log).toHaveBeenCalledWith( + expect.objectContaining({ + metrics: { + prompt_cached_tokens: 20, + prompt_tokens: 33, + }, + }), + ); + expect(llmSpan?.log).toHaveBeenLastCalledWith({ + metrics: { + completion_tokens: 40, + prompt_cache_creation_1h_tokens: 0, + prompt_cache_creation_5m_tokens: 3, + prompt_cached_tokens: 20, + prompt_tokens: 33, + tokens: 73, + }, + }); + }); + + it("omits invalid partial completion usage when the transcript is unavailable", async () => { + const startEvent = { + arguments: [ + { + prompt: "Test", + options: { model: "claude-3-5-sonnet-20241022" }, + }, + ], + }; + handlers.start(startEvent); + expect( + "includePartialMessages" in startEvent.arguments[0].options, + ).toBe(false); + + const internalSessionStart = (startEvent.arguments[0].options as any) + .hooks.SessionStart[0].hooks[0]; + await internalSessionStart( + { + cwd: "/tmp", + hook_event_name: "SessionStart", + session_id: "session_1", + transcript_path: "/tmp/missing.jsonl", + }, + undefined, + { signal: new AbortController().signal }, + ); + vi.mocked(iso.readFile!).mockRejectedValue(new Error("missing")); + + const stream = { + async *[Symbol.asyncIterator]() { + // The patcher is mocked; messages are delivered below. + }, + }; + handlers.end(Object.assign(startEvent, { result: stream })); + await streamPatcherMock.options?.onChunk?.({ + type: "stream_event", + event: { + type: "message_start", + message: { + id: "msg_missing", + usage: { + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 1, + }, + }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ + type: "stream_event", + event: { + type: "message_delta", + usage: { output_tokens: -1 }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ + type: "assistant", + message: { + content: [{ text: "Response", type: "text" }], + id: "msg_missing", + role: "assistant", + usage: { + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 1, + }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ type: "result" }); + await expect( + streamPatcherMock.options?.onComplete(), + ).resolves.toBeUndefined(); + + const llmSpanCallIndex = vi + .mocked(startSpan) + .mock.calls.findIndex( + ([args]) => + args && + typeof args === "object" && + "name" in args && + args.name === "anthropic.messages.create", + ); + const llmSpan = + vi.mocked(startSpan).mock.results[llmSpanCallIndex]?.value; + const metricLogs = vi + .mocked(llmSpan!.log) + .mock.calls.map((call: any[]) => call[0].metrics) + .filter(Boolean); + expect(metricLogs).toEqual([ + { + prompt_cached_tokens: 20, + prompt_tokens: 33, + }, + { + prompt_cache_creation_tokens: 3, + }, + ]); + }); + + it("keeps the LLM span when individual usage fields are unusable", async () => { + const startEvent = { + arguments: [ + { + prompt: "Test", + options: { model: "claude-3-5-sonnet-20241022" }, + }, + ], + }; + handlers.start(startEvent); + vi.mocked(iso.readFile!).mockRejectedValue(new Error("missing")); + + const stream = { + async *[Symbol.asyncIterator]() { + // The patcher is mocked; messages are delivered below. + }, + }; + handlers.end(Object.assign(startEvent, { result: stream })); + await streamPatcherMock.options?.onChunk?.({ + type: "assistant", + message: { + content: [{ text: "Response", type: "text" }], + id: "msg_null_cache", + model: "claude-3-5-sonnet-20241022", + role: "assistant", + // Bedrock, Vertex, and gateway-backed runs report `null` for the + // cache fields they do not populate. + usage: { + cache_creation: null, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + input_tokens: 10, + output_tokens: 40, + }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ type: "result" }); + await streamPatcherMock.options?.onComplete(); + + const llmSpanCallIndex = vi + .mocked(startSpan) + .mock.calls.findIndex( + ([args]) => + args && + typeof args === "object" && + "name" in args && + args.name === "anthropic.messages.create", + ); + expect(llmSpanCallIndex).toBeGreaterThan(-1); + const llmSpan = + vi.mocked(startSpan).mock.results[llmSpanCallIndex]?.value; + expect(llmSpan?.log).toHaveBeenCalledWith( + expect.objectContaining({ + metadata: { + model: "claude-3-5-sonnet-20241022", + provider: "anthropic", + }, + metrics: { prompt_tokens: 10 }, + output: [ + { + content: [{ text: "Response", type: "text" }], + role: "assistant", + }, + ], + }), + ); + }); + + it("layers partial stream usage over the assistant message usage", async () => { + const startEvent = { + arguments: [ + { + prompt: "Test", + options: { + includePartialMessages: true, + model: "claude-3-5-sonnet-20241022", + }, + }, + ], + }; + handlers.start(startEvent); + + const stream = { + async *[Symbol.asyncIterator]() { + // The patcher is mocked; messages are delivered below. + }, + }; + handlers.end(Object.assign(startEvent, { result: stream })); + await streamPatcherMock.options?.onChunk?.({ + type: "stream_event", + event: { + type: "message_start", + // No usable counts here, so the prompt-side totals must come from + // the assistant message rather than defaulting to zero. + message: { id: "msg_partial", usage: { input_tokens: null } }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ + type: "stream_event", + event: { type: "message_delta", usage: { output_tokens: 40 } }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ + type: "assistant", + message: { + content: [{ text: "Response", type: "text" }], + id: "msg_partial", + role: "assistant", + usage: { + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 40, + }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ type: "result" }); + await streamPatcherMock.options?.onComplete(); + + const llmSpanCallIndex = vi + .mocked(startSpan) + .mock.calls.findIndex( + ([args]) => + args && + typeof args === "object" && + "name" in args && + args.name === "anthropic.messages.create", + ); + const llmSpan = + vi.mocked(startSpan).mock.results[llmSpanCallIndex]?.value; + expect(llmSpan?.log).toHaveBeenCalledWith( + expect.objectContaining({ + metrics: { + completion_tokens: 40, + prompt_cache_creation_tokens: 3, + prompt_cached_tokens: 20, + prompt_tokens: 33, + tokens: 73, + }, + }), + ); + }); }); describe("error handler", () => { diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts index 8a88c5daf..c5ce305d4 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts +++ b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts @@ -1,7 +1,8 @@ import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; -import type { IsoChannelHandlers } from "../../isomorph"; +import iso, { type IsoChannelHandlers } from "../../isomorph"; +import { debugLogger } from "../../debug-logger"; import { startSpan as startBaseSpan } from "../../logger"; import type { Span } from "../../logger"; import { @@ -13,6 +14,8 @@ import { getCurrentUnixTimestamp } from "../../util"; import { extractAnthropicCacheTokens, finalizeAnthropicTokens, + toNumericMetrics, + type AnthropicTokenMetrics, } from "../../wrappers/anthropic-tokens-util"; import { claudeAgentSDKChannels } from "./claude-agent-sdk-channels"; import { CLAUDE_AGENT_SDK_SKIP_LOCAL_TOOL_HOOKS_OPTION } from "./claude-agent-sdk-instrumentation-constants"; @@ -33,6 +36,7 @@ import type { ClaudeAgentSDKMessage, ClaudeAgentSDKQueryOptions, ClaudeAgentSDKQueryParams, + ClaudeAgentSDKUsage, } from "../../vendor-sdk-types/claude-agent-sdk"; type ClaudeConversationMessage = { content: unknown; role: string }; @@ -48,8 +52,19 @@ type ParentSpanResolver = ( ) => Promise; type LLMSpanResult = { finalMessage: ClaudeConversationMessage | undefined; + span: Span; spanExport: string; }; +type PendingLLMUsage = { + messageId: string; + span: Span; + usage: ClaudeAgentSDKUsage; +}; +type TranscriptUsageState = { + pendingLlmUsageByContextKey: Map; + rootTranscriptPath?: string; + subagentTranscriptPathByToolUseId: Map; +}; type SubAgentDetails = { agentId?: string; agentType?: string; @@ -71,6 +86,13 @@ function llmParentKey(parentToolUseId: string | null): string { return parentToolUseId ?? ROOT_LLM_PARENT_KEY; } +function llmUsageContextKey( + parentToolUseId: string | null, + messageId: string, +): string { + return JSON.stringify([llmParentKey(parentToolUseId), messageId]); +} + function isSubAgentDelegationToolName(toolName: string): boolean { return toolName === "Agent" || toolName === "Task"; } @@ -211,20 +233,335 @@ function seedTaskToolUseIdMapping( } } -function extractUsageFromMessage( - message: ClaudeAgentSDKMessage, -): Record { - const metrics: Record = {}; +function tokenCount(value: unknown): number | undefined { + return typeof value === "number" && + Number.isFinite(value) && + Number.isInteger(value) && + value >= 0 + ? value + : undefined; +} - let usage: unknown; - if (message.type === "assistant") { - usage = message.message?.usage; - } else if (message.type === "result") { - usage = message.usage; +/** + * Snapshots the token fields we understand out of a provider usage object. + * + * Fields degrade individually: Bedrock, Vertex, and gateway-backed runs report + * `null` for cache fields they do not populate, and one unusable field must + * only cost that metric rather than the whole usage object. + */ +function copyUsage( + usage: unknown, + requireInputAndOutput = false, +): ClaudeAgentSDKUsage | undefined { + if (!usage || typeof usage !== "object") { + return undefined; } - if (!usage || typeof usage !== "object") { - return metrics; + const copy: ClaudeAgentSDKUsage = {}; + for (const key of [ + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ] as const) { + const value = tokenCount(Reflect.get(usage, key)); + if (value !== undefined) { + copy[key] = value; + } + } + + const cacheCreation = Reflect.get(usage, "cache_creation"); + if (cacheCreation && typeof cacheCreation === "object") { + const cacheCreationCopy: NonNullable< + ClaudeAgentSDKUsage["cache_creation"] + > = {}; + for (const key of [ + "ephemeral_5m_input_tokens", + "ephemeral_1h_input_tokens", + ] as const) { + const value = tokenCount(Reflect.get(cacheCreation, key)); + if (value !== undefined) { + cacheCreationCopy[key] = value; + } + } + if (Object.keys(cacheCreationCopy).length > 0) { + copy.cache_creation = cacheCreationCopy; + } + } + + if ( + requireInputAndOutput && + (copy.input_tokens === undefined || copy.output_tokens === undefined) + ) { + return undefined; + } + + return Object.keys(copy).length > 0 ? copy : undefined; +} + +/** Layers a newer usage snapshot over an older one, field by field. */ +function mergeUsage( + base: ClaudeAgentSDKUsage | undefined, + override: ClaudeAgentSDKUsage | undefined, +): ClaudeAgentSDKUsage | undefined { + if (!base || !override) { + return override ?? base; + } + + const cacheCreation = + base.cache_creation || override.cache_creation + ? { ...base.cache_creation, ...override.cache_creation } + : undefined; + return { + ...base, + ...override, + ...(cacheCreation && { cache_creation: cacheCreation }), + }; +} + +function parseTranscriptRowUsage( + line: string, + messageId: string, +): ClaudeAgentSDKUsage | undefined { + let row: unknown; + try { + row = JSON.parse(line); + } catch { + return undefined; + } + + if (!row || typeof row !== "object") { + return undefined; + } + if (getStringProperty(row, "type") !== "assistant") { + return undefined; + } + + const message = Reflect.get(row, "message"); + if (!message || typeof message !== "object") { + return undefined; + } + if (getStringProperty(message, "id") !== messageId) { + return undefined; + } + + return copyUsage(Reflect.get(message, "usage"), true); +} + +function parseTranscriptUsage( + bytes: Uint8Array, + messageIds: Set, +): Map { + const finalUsageByMessageId = new Map(); + if (messageIds.size === 0) { + return finalUsageByMessageId; + } + + const text = new TextDecoder().decode(bytes); + for (const messageId of messageIds) { + // Transcripts for `continue`/`resume` sessions routinely reach tens of + // megabytes, so seek the rows we actually need instead of parsing every + // row in the session on each query completion. + let searchTo = text.length; + while (searchTo >= 0) { + const match = text.lastIndexOf(messageId, searchTo); + if (match < 0) { + break; + } + + const lineStart = text.lastIndexOf("\n", match) + 1; + const lineEnd = text.indexOf("\n", match); + // Transcript rows for one provider request are successive snapshots, not + // separate model calls. The last valid row contains the final usage. + const usage = parseTranscriptRowUsage( + text.slice(lineStart, lineEnd < 0 ? text.length : lineEnd), + messageId, + ); + if (usage) { + finalUsageByMessageId.set(messageId, usage); + break; + } + + searchTo = lineStart - 1; + } + } + + return finalUsageByMessageId; +} + +function recoveredUsage( + pendingUsage: ClaudeAgentSDKUsage, + transcriptUsage: ClaudeAgentSDKUsage, +): ClaudeAgentSDKUsage | undefined { + const inputTokens = tokenCount( + transcriptUsage.input_tokens ?? pendingUsage.input_tokens, + ); + const outputTokens = tokenCount(transcriptUsage.output_tokens); + const cacheReadTokens = tokenCount( + transcriptUsage.cache_read_input_tokens ?? + pendingUsage.cache_read_input_tokens ?? + 0, + ); + const cacheCreationTokens = tokenCount( + transcriptUsage.cache_creation_input_tokens ?? + pendingUsage.cache_creation_input_tokens ?? + 0, + ); + + if ( + inputTokens === undefined || + outputTokens === undefined || + cacheReadTokens === undefined || + cacheCreationTokens === undefined + ) { + return undefined; + } + + const cacheCreation = + transcriptUsage.cache_creation ?? pendingUsage.cache_creation; + + return { + cache_creation_input_tokens: cacheCreationTokens, + cache_read_input_tokens: cacheReadTokens, + input_tokens: inputTokens, + output_tokens: outputTokens, + ...(cacheCreation && { cache_creation: { ...cacheCreation } }), + }; +} + +async function recoverUsageFromTranscript( + state: TranscriptUsageState, +): Promise { + const readFile = iso.readFile; + if (!readFile || state.pendingLlmUsageByContextKey.size === 0) { + return; + } + + const pendingMessageIds = new Set(); + for (const pending of state.pendingLlmUsageByContextKey.values()) { + pendingMessageIds.add(pending.messageId); + } + + const parentToolUseIdsByTranscriptPath = new Map< + string, + Set + >(); + const addTranscriptPath = ( + transcriptPath: string, + parentToolUseId: string | null, + ) => { + const parentToolUseIds = + parentToolUseIdsByTranscriptPath.get(transcriptPath) ?? new Set(); + parentToolUseIds.add(parentToolUseId); + parentToolUseIdsByTranscriptPath.set(transcriptPath, parentToolUseIds); + }; + + if (state.rootTranscriptPath) { + addTranscriptPath(state.rootTranscriptPath, null); + } + for (const [ + toolUseId, + transcriptPath, + ] of state.subagentTranscriptPathByToolUseId) { + addTranscriptPath(transcriptPath, toolUseId); + } + + const transcriptUsages = await Promise.all( + Array.from( + parentToolUseIdsByTranscriptPath, + async ([transcriptPath, parentToolUseIds]) => { + try { + return { + parentToolUseIds, + usageByMessageId: parseTranscriptUsage( + await readFile(transcriptPath), + pendingMessageIds, + ), + }; + } catch (error) { + debugLogger.debug( + "Could not recover Claude Agent SDK transcript usage", + error, + ); + return undefined; + } + }, + ), + ); + + for (const transcriptUsage of transcriptUsages) { + if (!transcriptUsage) { + continue; + } + + for (const parentToolUseId of transcriptUsage.parentToolUseIds) { + for (const [ + messageId, + usageFromTranscript, + ] of transcriptUsage.usageByMessageId) { + const contextKey = llmUsageContextKey(parentToolUseId, messageId); + const pending = state.pendingLlmUsageByContextKey.get(contextKey); + if (!pending) { + continue; + } + + const usage = recoveredUsage(pending.usage, usageFromTranscript); + if (!usage) { + continue; + } + + try { + pending.span.log({ metrics: extractUsage(usage, true) }); + state.pendingLlmUsageByContextKey.delete(contextKey); + } catch (error) { + debugLogger.debug( + "Could not update Claude Agent SDK span with transcript usage", + error, + ); + } + } + } + } +} + +// The CLI appends the final assistant row around the time the query stream +// completes, so give a transcript that is still missing rows a couple of very +// short chances to catch up before giving up on those spans' output tokens. +const TRANSCRIPT_RECOVERY_ATTEMPTS = 3; +const TRANSCRIPT_RECOVERY_RETRY_MS = 25; + +async function recoverUsageWithRetries( + state: TranscriptUsageState, +): Promise { + for (let attempt = 0; attempt < TRANSCRIPT_RECOVERY_ATTEMPTS; attempt++) { + if (attempt > 0) { + await new Promise((resolve) => + setTimeout(resolve, TRANSCRIPT_RECOVERY_RETRY_MS), + ); + } + + await recoverUsageFromTranscript(state); + if (state.pendingLlmUsageByContextKey.size === 0) { + return; + } + if ( + state.rootTranscriptPath === undefined && + state.subagentTranscriptPathByToolUseId.size === 0 + ) { + // No transcript to wait on, so retrying cannot resolve anything. + return; + } + } +} + +function extractUsage( + usage: ClaudeAgentSDKUsage | undefined, + includeOutput: boolean, + omitLegacyCacheCreationMetric = false, +): Record { + const metrics: AnthropicTokenMetrics = {}; + if (!usage) { + return {}; } const inputTokens = getNumberProperty(usage, "input_tokens"); @@ -232,28 +569,56 @@ function extractUsageFromMessage( metrics.prompt_tokens = inputTokens; } - const outputTokens = getNumberProperty(usage, "output_tokens"); - if (outputTokens !== undefined) { - metrics.completion_tokens = outputTokens; + if (includeOutput) { + const outputTokens = getNumberProperty(usage, "output_tokens"); + if (outputTokens !== undefined) { + metrics.completion_tokens = outputTokens; + } } const cacheReadTokens = getNumberProperty(usage, "cache_read_input_tokens") || 0; const cacheCreationTokens = getNumberProperty(usage, "cache_creation_input_tokens") || 0; + Object.assign( + metrics, + extractAnthropicCacheTokens(cacheReadTokens, cacheCreationTokens), + ); - if (cacheReadTokens > 0 || cacheCreationTokens > 0) { - Object.assign( - metrics, - extractAnthropicCacheTokens(cacheReadTokens, cacheCreationTokens), - ); + const cacheCreation5mTokens = getNumberProperty( + usage.cache_creation, + "ephemeral_5m_input_tokens", + ); + const cacheCreation1hTokens = getNumberProperty( + usage.cache_creation, + "ephemeral_1h_input_tokens", + ); + if (cacheCreation5mTokens !== undefined) { + metrics.prompt_cache_creation_5m_tokens = cacheCreation5mTokens; + } + if (cacheCreation1hTokens !== undefined) { + metrics.prompt_cache_creation_1h_tokens = cacheCreation1hTokens; } - if (Object.keys(metrics).length > 0) { - Object.assign(metrics, finalizeAnthropicTokens(metrics)); + if (Object.keys(metrics).length === 0) { + return {}; + } + + // `finalizeAnthropicTokens` drops the aggregate cache-creation metric when the + // per-TTL breakdown is present, so the finalized object replaces `metrics` + // rather than being merged back over it. + const finalized = finalizeAnthropicTokens(metrics); + if (metrics.completion_tokens === undefined) { + // A total is only meaningful once both halves are known. + delete finalized.tokens; + } + if (omitLegacyCacheCreationMetric) { + // Span metrics can only be added, never removed, so hold the aggregate back + // until we know a per-TTL breakdown will not arrive from the transcript. + delete finalized.prompt_cache_creation_tokens; } - return metrics; + return toNumericMetrics(finalized); } function buildLLMInput( @@ -325,6 +690,8 @@ async function createLLMSpanForMessages( options: ClaudeAgentSDKQueryOptions, startTime: number, parentSpan: string, + usage: ClaudeAgentSDKUsage | undefined, + hasFinalOutputUsage: boolean, existingSpan?: Span, ): Promise { if (messages.length === 0) { @@ -332,12 +699,18 @@ async function createLLMSpanForMessages( } const lastMessage = messages[messages.length - 1]; - if (lastMessage.type !== "assistant" || !lastMessage.message?.usage) { + // Every assistant message is one provider request and must produce one `llm` + // span. Unusable usage costs metrics, never the span or its payloads. + if (lastMessage.type !== "assistant") { return undefined; } - const model = lastMessage.message.model || options.model; - const usage = extractUsageFromMessage(lastMessage); + const model = lastMessage.message?.model || options.model; + const metrics = extractUsage( + usage, + hasFinalOutputUsage, + !hasFinalOutputUsage, + ); const input = buildLLMInput(promptMessages, conversationHistory); const outputs = messages .map((m) => @@ -368,8 +741,8 @@ async function createLLMSpanForMessages( span.log({ input, - metadata: model ? { model } : undefined, - metrics: usage, + metadata: { ...(model && { model }), provider: "anthropic" }, + ...(Object.keys(metrics).length > 0 ? { metrics } : {}), output: outputs, }); @@ -383,6 +756,7 @@ async function createLLMSpanForMessages( return { finalMessage, + span, spanExport, }; } @@ -500,6 +874,8 @@ function createToolTracingHooks( subAgentDetailsByToolUseId: Map, subAgentSpans: Map, endedSubAgentSpans: Set, + transcriptUsageState: TranscriptUsageState, + taskIdToToolUseId: Map, ): { postToolUse: ClaudeAgentSDKHookCallback; postToolUseFailure: ClaudeAgentSDKHookCallback; @@ -749,6 +1125,14 @@ function createToolTracingHooks( toolUseId: toolUseID, }, ); + if (input.agent_transcript_path) { + const parentToolUseId = + taskIdToToolUseId.get(input.agent_id) ?? toolUseID; + transcriptUsageState.subagentTranscriptPathByToolUseId.set( + parentToolUseId, + input.agent_transcript_path, + ); + } const subAgentSpan = subAgentSpans.get(toolUseID); if (!subAgentSpan || endedSubAgentSpans.has(toolUseID)) { return {}; @@ -756,9 +1140,6 @@ function createToolTracingHooks( const metadata = { ...subAgentDetailsToMetadata(details), - ...(input.agent_transcript_path && { - "claude_agent_sdk.agent_transcript_path": input.agent_transcript_path, - }), "claude_agent_sdk.stop_hook_active": input.stop_hook_active, }; @@ -793,6 +1174,8 @@ function injectTracingHooks( subAgentDetailsByToolUseId: Map, subAgentSpans: Map, endedSubAgentSpans: Set, + transcriptUsageState: TranscriptUsageState, + taskIdToToolUseId: Map, ): ClaudeAgentSDKQueryOptions { const { preToolUse, @@ -809,7 +1192,26 @@ function injectTracingHooks( subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans, + transcriptUsageState, + taskIdToToolUseId, ); + // SessionStart can occur before programmatic hooks are registered in the + // supported SDK versions. UserPromptSubmit carries the same base hook fields + // after registration, while SessionEnd remains a useful final fallback. + const captureRootTranscriptPath: ClaudeAgentSDKHookCallback = async ( + input, + ) => { + if ( + (input.hook_event_name === "SessionStart" || + input.hook_event_name === "SessionEnd" || + input.hook_event_name === "UserPromptSubmit") && + input.agent_id === undefined && + typeof input.transcript_path === "string" + ) { + transcriptUsageState.rootTranscriptPath = input.transcript_path; + } + return {}; + }; const existingHooks = options.hooks ?? {}; @@ -817,6 +1219,24 @@ function injectTracingHooks( ...options, hooks: { ...existingHooks, + SessionStart: [ + ...(existingHooks.SessionStart ?? []), + { + hooks: [captureRootTranscriptPath], + } satisfies ClaudeAgentSDKHookCallbackMatcher, + ], + SessionEnd: [ + ...(existingHooks.SessionEnd ?? []), + { + hooks: [captureRootTranscriptPath], + } satisfies ClaudeAgentSDKHookCallbackMatcher, + ], + UserPromptSubmit: [ + ...(existingHooks.UserPromptSubmit ?? []), + { + hooks: [captureRootTranscriptPath], + } satisfies ClaudeAgentSDKHookCallbackMatcher, + ], PostToolUse: [ ...(existingHooks.PostToolUse ?? []), { hooks: [postToolUse] } satisfies ClaudeAgentSDKHookCallbackMatcher, @@ -848,8 +1268,8 @@ function injectTracingHooks( } type QueryState = { - accumulatedOutputTokens: number; activeLlmSpansByParentToolUse: Map; + activePartialMessageIdByParentKey: Map; activeToolSpans: Map; conversationHistoryByParentKey: Map; capturedPromptMessages: ClaudeAgentSDKMessage[] | undefined; @@ -857,6 +1277,7 @@ type QueryState = { currentMessageStartTime: number; currentMessages: ClaudeAgentSDKMessage[]; endedSubAgentSpans: Set; + finalOutputUsageMessageIds: Set; finalResults: ClaudeConversationMessage[]; options: ClaudeAgentSDKQueryOptions; originalPrompt: string | AsyncIterable | undefined; @@ -872,6 +1293,8 @@ type QueryState = { latestLlmParentBySubAgentToolUse: Map; latestRootLlmParentRef: { value: string | undefined }; toolUseToParent: Map; + transcriptUsageState: TranscriptUsageState; + usageByMessageId: Map; localToolContext: ClaudeAgentSDKLocalToolContext; }; @@ -931,6 +1354,17 @@ async function finalizeCurrentMessageGroup(state: QueryState): Promise { } } const existingLlmSpan = state.activeLlmSpansByParentToolUse.get(parentKey); + const lastMessage = state.currentMessages[state.currentMessages.length - 1]; + const messageId = lastMessage?.message?.id; + // Stream events carry the freshest counts, but they can be partial (a + // `message_delta` reports only `output_tokens`), so layer them over the + // assistant message's own usage rather than replacing it. + const usage = mergeUsage( + copyUsage(lastMessage?.message?.usage), + messageId ? state.usageByMessageId.get(messageId) : undefined, + ); + const hasFinalOutputUsage = + messageId !== undefined && state.finalOutputUsageMessageIds.has(messageId); const llmSpanResult = await createLLMSpanForMessages( state.currentMessages, @@ -939,6 +1373,8 @@ async function finalizeCurrentMessageGroup(state: QueryState): Promise { state.options, state.currentMessageStartTime, parentSpan, + usage, + hasFinalOutputUsage, existingLlmSpan, ); @@ -956,6 +1392,17 @@ async function finalizeCurrentMessageGroup(state: QueryState): Promise { conversationHistory.push(llmSpanResult.finalMessage); state.finalResults.push(llmSpanResult.finalMessage); } + + if (messageId && !hasFinalOutputUsage && usage) { + state.transcriptUsageState.pendingLlmUsageByContextKey.set( + llmUsageContextKey(parentToolUseId, messageId), + { + messageId, + span: llmSpanResult.span, + usage, + }, + ); + } } // Keep the active LLM parent visible until the finalized exported parent @@ -963,10 +1410,17 @@ async function finalizeCurrentMessageGroup(state: QueryState): Promise { // fall back to the broader sub-agent task span instead of the LLM span. state.activeLlmSpansByParentToolUse.delete(parentKey); - const lastMessage = state.currentMessages[state.currentMessages.length - 1]; - if (lastMessage?.message?.usage) { - state.accumulatedOutputTokens += - getNumberProperty(lastMessage.message.usage, "output_tokens") || 0; + if (messageId) { + state.usageByMessageId.delete(messageId); + state.finalOutputUsageMessageIds.delete(messageId); + for (const [ + parent, + activeMessageId, + ] of state.activePartialMessageIdByParentKey) { + if (activeMessageId === messageId) { + state.activePartialMessageIdByParentKey.delete(parent); + } + } } state.currentMessages.length = 0; @@ -1255,10 +1709,62 @@ async function maybeHandleTaskLifecycleMessage( return true; } +function handlePartialUsageMessage( + state: QueryState, + message: ClaudeAgentSDKMessage, +): boolean { + if (message.type !== "stream_event") { + return false; + } + + const event = message.event; + if (!event || typeof event !== "object") { + return true; + } + + const parentKey = llmParentKey(message.parent_tool_use_id ?? null); + if (event.type === "message_start") { + const messageId = event.message?.id; + const usage = copyUsage(event.message?.usage); + if (messageId) { + state.activePartialMessageIdByParentKey.set(parentKey, messageId); + if (usage) { + state.usageByMessageId.set(messageId, usage); + } + } + return true; + } + + const messageId = state.activePartialMessageIdByParentKey.get(parentKey); + if (!messageId) { + return true; + } + + if (event.type === "message_delta") { + const update = copyUsage(event.usage); + if (update) { + const usage = state.usageByMessageId.get(messageId) ?? {}; + Object.assign(usage, update); + state.usageByMessageId.set(messageId, usage); + if (update.output_tokens !== undefined) { + state.finalOutputUsageMessageIds.add(messageId); + } + } + } else if (event.type === "message_stop") { + state.activePartialMessageIdByParentKey.delete(parentKey); + } + + return true; +} + async function handleStreamMessage( state: QueryState, message: ClaudeAgentSDKMessage, ): Promise { + if (handlePartialUsageMessage(state, message)) { + return; + } + maybeTrackToolUseContext(state, message); if (await maybeHandleTaskLifecycleMessage(state, message)) { return; @@ -1317,45 +1823,10 @@ async function handleStreamMessage( state.currentMessages.push(message); } - if (message.type !== "result" || !message.usage) { + if (message.type !== "result") { return; } - const finalUsageMetrics = extractUsageFromMessage(message); - if ( - state.currentMessages.length > 0 && - finalUsageMetrics.completion_tokens !== undefined - ) { - const lastMessage = state.currentMessages[state.currentMessages.length - 1]; - if (lastMessage?.message?.usage) { - const adjustedTokens = - finalUsageMetrics.completion_tokens - state.accumulatedOutputTokens; - if (adjustedTokens >= 0) { - lastMessage.message.usage.output_tokens = adjustedTokens; - } - - const resultUsage = message.usage; - if (resultUsage && typeof resultUsage === "object") { - const cacheReadTokens = getNumberProperty( - resultUsage, - "cache_read_input_tokens", - ); - if (cacheReadTokens !== undefined) { - lastMessage.message.usage.cache_read_input_tokens = cacheReadTokens; - } - - const cacheCreationTokens = getNumberProperty( - resultUsage, - "cache_creation_input_tokens", - ); - if (cacheCreationTokens !== undefined) { - lastMessage.message.usage.cache_creation_input_tokens = - cacheCreationTokens; - } - } - } - } - const metadata: Record = {}; if (message.num_turns !== undefined) { metadata.num_turns = message.num_turns; @@ -1372,6 +1843,39 @@ async function finalizeQuerySpan(state: QueryState): Promise { try { await finalizeCurrentMessageGroup(state); + try { + await recoverUsageWithRetries(state.transcriptUsageState); + } catch (error) { + debugLogger.debug( + "Could not recover Claude Agent SDK transcript usage", + error, + ); + } + for (const pending of state.transcriptUsageState.pendingLlmUsageByContextKey.values()) { + const cacheCreationTokens = pending.usage.cache_creation_input_tokens; + const hasCacheCreationBreakdown = + pending.usage.cache_creation?.ephemeral_5m_input_tokens !== undefined || + pending.usage.cache_creation?.ephemeral_1h_input_tokens !== undefined; + if ( + hasCacheCreationBreakdown || + cacheCreationTokens === undefined || + cacheCreationTokens === 0 + ) { + continue; + } + + try { + pending.span.log({ + metrics: extractAnthropicCacheTokens(0, cacheCreationTokens), + }); + } catch (error) { + debugLogger.debug( + "Could not finalize Claude Agent SDK fallback cache usage", + error, + ); + } + } + state.span.log({ output: state.finalResults.length > 0 @@ -1394,6 +1898,12 @@ async function finalizeQuerySpan(state: QueryState): Promise { llmSpan.end(); } state.activeLlmSpansByParentToolUse.clear(); + state.activePartialMessageIdByParentKey.clear(); + state.finalOutputUsageMessageIds.clear(); + state.usageByMessageId.clear(); + state.transcriptUsageState.pendingLlmUsageByContextKey.clear(); + state.transcriptUsageState.subagentTranscriptPathByToolUseId.clear(); + state.transcriptUsageState.rootTranscriptPath = undefined; for (const toolSpan of state.activeToolSpans.values()) { toolSpan.end(); @@ -1509,6 +2019,10 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { >(); const promptSourcePriorityByParentKey = new Map(); const localToolContext = createClaudeLocalToolContext(); + const transcriptUsageState: TranscriptUsageState = { + pendingLlmUsageByContextKey: new Map(), + subagentTranscriptPathByToolUseId: new Map(), + }; const { hasLocalToolHandlers, localToolHookNames } = prepareLocalToolHandlersInMcpServers(options.mcpServers); const skipLocalToolHooks = @@ -1570,14 +2084,16 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans, + transcriptUsageState, + taskIdToToolUseId, ); params.options = optionsWithHooks; event.arguments[0] = params; spans.set(event, { - accumulatedOutputTokens: 0, activeLlmSpansByParentToolUse, + activePartialMessageIdByParentKey: new Map(), activeToolSpans, conversationHistoryByParentKey, capturedPromptMessages, @@ -1585,6 +2101,7 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { currentMessageStartTime: startTime, currentMessages: [], endedSubAgentSpans, + finalOutputUsageMessageIds: new Set(), finalResults: [], options: optionsWithHooks, originalPrompt, @@ -1600,6 +2117,8 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { latestLlmParentBySubAgentToolUse, latestRootLlmParentRef, toolUseToParent, + transcriptUsageState, + usageByMessageId: new Map(), localToolContext, }); }, diff --git a/js/src/instrumentation/plugins/github-copilot-plugin.ts b/js/src/instrumentation/plugins/github-copilot-plugin.ts index 5a1963a2e..8fa156cfc 100644 --- a/js/src/instrumentation/plugins/github-copilot-plugin.ts +++ b/js/src/instrumentation/plugins/github-copilot-plugin.ts @@ -39,7 +39,7 @@ export function extractMetricsFromUsage(usage: GitHubCopilotUsageData): { metrics: AnthropicTokenMetrics; metadata: Record; } { - const metrics: AnthropicTokenMetrics = { + const rawMetrics: AnthropicTokenMetrics = { prompt_tokens: usage.inputTokens, completion_tokens: usage.outputTokens, ...extractAnthropicCacheTokens( @@ -49,11 +49,13 @@ export function extractMetricsFromUsage(usage: GitHubCopilotUsageData): { }; if (usage.reasoningTokens !== undefined) { - metrics.completion_reasoning_tokens = usage.reasoningTokens; - metrics.reasoning_tokens = usage.reasoningTokens; + rawMetrics.completion_reasoning_tokens = usage.reasoningTokens; + rawMetrics.reasoning_tokens = usage.reasoningTokens; } - Object.assign(metrics, finalizeAnthropicTokens(metrics)); + // Use the returned object: finalization can drop cache-creation metrics, and + // merging it back over `rawMetrics` would keep them. + const metrics = finalizeAnthropicTokens(rawMetrics); const metadata: Record = { model: usage.model, diff --git a/js/src/vendor-sdk-types/anthropic.ts b/js/src/vendor-sdk-types/anthropic.ts index 278e7f392..89cdc644f 100644 --- a/js/src/vendor-sdk-types/anthropic.ts +++ b/js/src/vendor-sdk-types/anthropic.ts @@ -194,10 +194,21 @@ export interface AnthropicUsage { output_tokens: number; cache_read_input_tokens?: number; cache_creation_input_tokens?: number; + cache_creation?: AnthropicCacheCreationUsage | null; server_tool_use?: AnthropicServerToolUseUsage; [key: string]: unknown; } +/** + * Per-TTL breakdown of cache-write tokens. Only recent Anthropic SDK versions + * report this; older ones expose just `cache_creation_input_tokens`. + */ +export interface AnthropicCacheCreationUsage { + ephemeral_5m_input_tokens?: number; + ephemeral_1h_input_tokens?: number; + [key: string]: unknown; +} + export interface AnthropicServerToolUseUsage { web_search_requests?: number; [key: string]: unknown; diff --git a/js/src/vendor-sdk-types/claude-agent-sdk.ts b/js/src/vendor-sdk-types/claude-agent-sdk.ts index c576993bf..c558ae2ac 100644 --- a/js/src/vendor-sdk-types/claude-agent-sdk.ts +++ b/js/src/vendor-sdk-types/claude-agent-sdk.ts @@ -5,11 +5,24 @@ */ // Shared usage shape used in message and result -interface Usage { +export interface ClaudeAgentSDKUsage { input_tokens?: number; output_tokens?: number; cache_read_input_tokens?: number; cache_creation_input_tokens?: number; + cache_creation?: { + ephemeral_5m_input_tokens?: number; + ephemeral_1h_input_tokens?: number; + }; +} + +interface ClaudeAgentSDKRawStreamEvent { + type?: string; + message?: { + id?: string; + usage?: ClaudeAgentSDKUsage; + }; + usage?: ClaudeAgentSDKUsage; } interface TaskUsage { @@ -37,10 +50,11 @@ export interface ClaudeAgentSDKMessage { role?: string; content?: unknown; model?: string; - usage?: Usage; + usage?: ClaudeAgentSDKUsage; }; + event?: ClaudeAgentSDKRawStreamEvent; parent_tool_use_id?: string | null; - usage?: Usage | TaskUsage; + usage?: ClaudeAgentSDKUsage | TaskUsage; num_turns?: number; session_id?: string; task_id?: string; @@ -75,6 +89,7 @@ export interface ClaudeAgentSDKQueryOptions { debug?: boolean; agentName?: string; instructions?: string; + includePartialMessages?: boolean; mcpServers?: ClaudeAgentSDKMcpServersConfig; hooks?: Record; [key: string]: unknown; @@ -107,6 +122,15 @@ export interface ClaudeAgentSDKModule { export type ClaudeAgentSDKHookCallback = ( input: + | (BaseHookInput & { + hook_event_name: "SessionStart"; + }) + | (BaseHookInput & { + hook_event_name: "SessionEnd"; + }) + | (BaseHookInput & { + hook_event_name: "UserPromptSubmit"; + }) | (BaseHookInput & { hook_event_name: "PreToolUse"; tool_name: string; diff --git a/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts b/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts index 5c12e8b39..d483ed02b 100644 --- a/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts +++ b/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts @@ -9,6 +9,7 @@ import { import { extractAnthropicCacheTokens, finalizeAnthropicTokens, + toNumericMetrics, } from "../../anthropic-tokens-util"; import { processInputAttachments } from "../../attachment-utils"; @@ -129,7 +130,9 @@ function normalizeUsageMetrics( ); Object.assign(metrics, cacheTokens); - Object.assign(metrics, finalizeAnthropicTokens(metrics)); + // Use the returned object: finalization can drop cache-creation metrics, + // and merging it back over `metrics` would keep them. + return toNumericMetrics(finalizeAnthropicTokens(metrics)); } } diff --git a/js/src/wrappers/anthropic-tokens-util.test.ts b/js/src/wrappers/anthropic-tokens-util.test.ts new file mode 100644 index 000000000..567e82057 --- /dev/null +++ b/js/src/wrappers/anthropic-tokens-util.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { + finalizeAnthropicTokens, + toNumericMetrics, +} from "./anthropic-tokens-util"; + +describe("finalizeAnthropicTokens", () => { + it("includes legacy aggregate cache creation tokens", () => { + expect( + finalizeAnthropicTokens({ + completion_tokens: 48, + prompt_cache_creation_tokens: 199, + prompt_cached_tokens: 24_243, + prompt_tokens: 8, + }), + ).toMatchObject({ + prompt_tokens: 24_450, + tokens: 24_498, + }); + }); + + it("uses TTL cache creation tokens instead of the legacy aggregate", () => { + const metrics = finalizeAnthropicTokens({ + completion_tokens: 48, + prompt_cache_creation_1h_tokens: 0, + prompt_cache_creation_5m_tokens: 199, + prompt_cache_creation_tokens: 999, + prompt_cached_tokens: 24_243, + prompt_tokens: 8, + }); + + expect(metrics).toMatchObject({ + prompt_tokens: 24_450, + tokens: 24_498, + }); + expect(metrics.prompt_cache_creation_tokens).toBeUndefined(); + }); + + it("drops the aggregate for callers that keep the returned object", () => { + const raw = { + completion_tokens: 48, + prompt_cache_creation_5m_tokens: 199, + prompt_cache_creation_tokens: 199, + prompt_tokens: 8, + }; + + // Merging the result back over the input would keep both representations, + // which Anthropic spans must never carry. + expect( + Object.assign({ ...raw }, finalizeAnthropicTokens(raw)), + ).toHaveProperty("prompt_cache_creation_tokens"); + expect(finalizeAnthropicTokens(raw)).not.toHaveProperty( + "prompt_cache_creation_tokens", + ); + }); +}); + +describe("toNumericMetrics", () => { + it("drops unset metrics", () => { + expect( + toNumericMetrics({ + completion_tokens: 0, + prompt_cache_creation_tokens: undefined, + prompt_tokens: 8, + }), + ).toEqual({ completion_tokens: 0, prompt_tokens: 8 }); + }); +}); diff --git a/js/src/wrappers/anthropic-tokens-util.ts b/js/src/wrappers/anthropic-tokens-util.ts index f5cd84aac..9be227099 100644 --- a/js/src/wrappers/anthropic-tokens-util.ts +++ b/js/src/wrappers/anthropic-tokens-util.ts @@ -9,23 +9,63 @@ export interface AnthropicTokenMetrics { completion_tokens?: number; prompt_cached_tokens?: number; prompt_cache_creation_tokens?: number; + prompt_cache_creation_5m_tokens?: number; + prompt_cache_creation_1h_tokens?: number; tokens?: number; [key: string]: number | undefined; } +/** + * Rolls cache tokens back into `prompt_tokens`/`tokens` and reduces the + * cache-creation metrics to a single representation. + * + * Callers MUST use the returned object rather than `Object.assign`-ing it back + * over the input: when a per-TTL breakdown is present this drops the aggregate + * `prompt_cache_creation_tokens`, and a merge back over the input would keep + * both representations on the span. + */ export function finalizeAnthropicTokens( metrics: AnthropicTokenMetrics, ): AnthropicTokenMetrics { + const hasSplitCacheCreationTokens = + metrics.prompt_cache_creation_5m_tokens !== undefined || + metrics.prompt_cache_creation_1h_tokens !== undefined; + const splitCacheCreationTokens = + (metrics.prompt_cache_creation_5m_tokens || 0) + + (metrics.prompt_cache_creation_1h_tokens || 0); + // The split is an alternative representation of the aggregate, not extra + // tokens, and only one of the two is emitted — so `prompt_tokens` is sized + // from whichever representation the span will carry. + const effectiveCacheCreationTokens = hasSplitCacheCreationTokens + ? splitCacheCreationTokens + : metrics.prompt_cache_creation_tokens || 0; const prompt_tokens = (metrics.prompt_tokens || 0) + (metrics.prompt_cached_tokens || 0) + - (metrics.prompt_cache_creation_tokens || 0); + effectiveCacheCreationTokens; - return { + const finalized = { ...metrics, prompt_tokens, tokens: prompt_tokens + (metrics.completion_tokens || 0), }; + // Anthropic spans must carry exactly one cache-creation representation, and + // the per-TTL breakdown wins whenever the provider reported it. + if (hasSplitCacheCreationTokens) { + delete finalized.prompt_cache_creation_tokens; + } + return finalized; +} + +/** Drops unset metrics so the result can be logged as span metrics. */ +export function toNumericMetrics( + metrics: AnthropicTokenMetrics, +): Record { + return Object.fromEntries( + Object.entries(metrics).filter( + (entry): entry is [string, number] => entry[1] !== undefined, + ), + ); } export function extractAnthropicCacheTokens( From fbac22509845c911aaaec58f32ed554f395b98fb Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:46:48 +0000 Subject: [PATCH 2/2] Update PR #2348 --- ...aude-agent-sdk-v0-auto-hook.span-tree.json | 103 ++--- ...laude-agent-sdk-v0-auto-hook.span-tree.txt | 103 ++--- ...ent-sdk-v0-latest-auto-hook.span-tree.json | 108 ++--- ...gent-sdk-v0-latest-auto-hook.span-tree.txt | 108 ++--- ...agent-sdk-v0-latest-wrapped.span-tree.json | 108 ++--- ...-agent-sdk-v0-latest-wrapped.span-tree.txt | 108 ++--- ...claude-agent-sdk-v0-wrapped.span-tree.json | 103 ++--- .../claude-agent-sdk-v0-wrapped.span-tree.txt | 103 ++--- .../assertions.ts | 190 +++----- .../scenario.impl.mjs | 198 --------- .../plugins/anthropic-plugin.test.ts | 4 +- .../plugins/anthropic-plugin.ts | 4 +- .../plugins/claude-agent-sdk-plugin.test.ts | 174 +++----- .../plugins/claude-agent-sdk-plugin.ts | 420 +----------------- js/src/vendor-sdk-types/claude-agent-sdk.ts | 9 - js/src/wrappers/anthropic-tokens-util.test.ts | 21 +- js/src/wrappers/anthropic-tokens-util.ts | 30 +- 17 files changed, 435 insertions(+), 1459 deletions(-) diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json index 020a98518..3054e8b65 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json @@ -223,13 +223,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 171, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 18650, - "prompt_tokens": 18660, - "tokens": 18831 } }, { @@ -279,14 +272,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 190, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 184, - "prompt_cached_tokens": 18650, - "prompt_tokens": 18844, - "tokens": 19034 } } ], @@ -321,6 +306,14 @@ "num_turns": 1, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 190, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 184, + "prompt_cached_tokens": 18650, + "prompt_tokens": 18844, + "tokens": 19034 } } ], @@ -379,13 +372,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 215, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 18811, - "prompt_tokens": 18821, - "tokens": 19036 } }, { @@ -434,13 +420,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 106, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 26759, - "prompt_tokens": 26762, - "tokens": 26868 } }, { @@ -591,14 +570,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 82, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 307, - "prompt_cached_tokens": 18811, - "prompt_tokens": 19126, - "tokens": 19208 } } ], @@ -620,6 +591,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 297, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 19118, + "prompt_cached_tokens": 18811, + "prompt_tokens": 37947, + "tokens": 38244 } } ], @@ -678,13 +657,6 @@ "metadata": { "model": "claude-sonnet-4-5-20250929", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 169, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 18711, - "prompt_tokens": 18721, - "tokens": 18890 } }, { @@ -732,13 +704,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 76, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 17229, - "prompt_tokens": 17232, - "tokens": 17308 } }, { @@ -883,14 +848,6 @@ "metadata": { "model": "claude-sonnet-4-5-20250929", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 166, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 250, - "prompt_cached_tokens": 18711, - "prompt_tokens": 18969, - "tokens": 19135 } } ], @@ -913,6 +870,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 335, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18961, + "prompt_cached_tokens": 18711, + "prompt_tokens": 37690, + "tokens": 38025 } } ], @@ -971,12 +936,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 0, - "prompt_cached_tokens": 18784, - "prompt_tokens": 18794 } }, { @@ -1054,12 +1013,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 199, - "prompt_cached_tokens": 18784, - "prompt_tokens": 18991 } } ], @@ -1078,6 +1031,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 233, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37785, + "tokens": 38018 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt index eaa5b4c46..775d02457 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt @@ -191,6 +191,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 190, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 184, + │ "prompt_cached_tokens": 18650, + │ "prompt_tokens": 18844, + │ "tokens": 19034 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -227,13 +235,6 @@ span_tree: │ │ "model": "claude-haiku-4-5-20251001", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 171, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 18650, - │ │ "prompt_tokens": 18660, - │ │ "tokens": 18831 - │ │ } │ └── anthropic.messages.create [llm] │ input: [ │ { @@ -279,14 +280,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 190, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 184, - │ "prompt_cached_tokens": 18650, - │ "prompt_tokens": 18844, - │ "tokens": 19034 - │ } ├── claude-agent-subagent-operation │ metadata: { │ "operation": "subagent", @@ -312,6 +305,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 297, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 19118, + │ "prompt_cached_tokens": 18811, + │ "prompt_tokens": 37947, + │ "tokens": 38244 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -353,13 +354,6 @@ span_tree: │ │ "model": "claude-haiku-4-5-20251001", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 215, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 18811, - │ │ "prompt_tokens": 18821, - │ │ "tokens": 19036 - │ │ } │ ├── tool: Agent [tool] │ │ input: { │ │ "description": "Add 15 and 27 using calculator", @@ -461,13 +455,6 @@ span_tree: │ │ │ "model": "claude-haiku-4-5-20251001", │ │ │ "provider": "anthropic" │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 106, - │ │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ │ "prompt_cache_creation_5m_tokens": 26759, - │ │ │ "prompt_tokens": 26762, - │ │ │ "tokens": 26868 - │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { │ │ "a": 15, @@ -539,14 +526,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 82, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 307, - │ "prompt_cached_tokens": 18811, - │ "prompt_tokens": 19126, - │ "tokens": 19208 - │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { │ "operation": "subagent-built-in-tool", @@ -573,6 +552,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 335, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 18961, + │ "prompt_cached_tokens": 18711, + │ "prompt_tokens": 37690, + │ "tokens": 38025 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -614,13 +601,6 @@ span_tree: │ │ "model": "claude-sonnet-4-5-20250929", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 169, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 18711, - │ │ "prompt_tokens": 18721, - │ │ "tokens": 18890 - │ │ } │ ├── tool: Agent [tool] │ │ input: { │ │ "description": "echo greeting", @@ -721,13 +701,6 @@ span_tree: │ │ │ "model": "claude-haiku-4-5-20251001", │ │ │ "provider": "anthropic" │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 76, - │ │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ │ "prompt_cache_creation_5m_tokens": 17229, - │ │ │ "prompt_tokens": 17232, - │ │ │ "tokens": 17308 - │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { │ │ "command": "echo hello", @@ -797,14 +770,6 @@ span_tree: │ "model": "claude-sonnet-4-5-20250929", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 166, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 250, - │ "prompt_cached_tokens": 18711, - │ "prompt_tokens": 18969, - │ "tokens": 19135 - │ } └── claude-agent-failure-operation metadata: { "operation": "failure", @@ -827,6 +792,14 @@ span_tree: "permissionMode": "bypassPermissions", "session_id": "" } + metrics: { + "completion_tokens": 233, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37785, + "tokens": 38018 + } ├── anthropic.messages.create [llm] │ input: [ │ { @@ -868,12 +841,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 0, - │ "prompt_cached_tokens": 18784, - │ "prompt_tokens": 18794 - │ } ├── tool: calculator/calculator [tool] │ input: { │ "a": 2, @@ -939,9 +906,3 @@ span_tree: "model": "claude-haiku-4-5-20251001", "provider": "anthropic" } - metrics: { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 199, - "prompt_cached_tokens": 18784, - "prompt_tokens": 18991 - } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json index 54de1f69c..bcc7566db 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json @@ -224,14 +224,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 475, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 0, - "prompt_cached_tokens": 24110, - "prompt_tokens": 24120, - "tokens": 24595 } }, { @@ -281,14 +273,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 148, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 488, - "prompt_cached_tokens": 24110, - "prompt_tokens": 24608, - "tokens": 24756 } } ], @@ -323,6 +307,14 @@ "num_turns": 1, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 148, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 488, + "prompt_cached_tokens": 24110, + "prompt_tokens": 24608, + "tokens": 24756 } } ], @@ -381,14 +373,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 234, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 0, - "prompt_cached_tokens": 24271, - "prompt_tokens": 24281, - "tokens": 24515 } }, { @@ -437,14 +421,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 106, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 2201, - "prompt_cached_tokens": 16794, - "prompt_tokens": 18998, - "tokens": 19104 } }, { @@ -617,14 +593,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 60, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 331, - "prompt_cached_tokens": 24271, - "prompt_tokens": 24610, - "tokens": 24670 } } ], @@ -646,6 +614,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 294, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 331, + "prompt_cached_tokens": 48542, + "prompt_tokens": 48891, + "tokens": 49185 } } ], @@ -704,14 +680,6 @@ "metadata": { "model": "claude-sonnet-4-5-20250929", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 259, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 0, - "prompt_cached_tokens": 24171, - "prompt_tokens": 24181, - "tokens": 24440 } }, { @@ -759,14 +727,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 74, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 1359, - "prompt_cached_tokens": 4548, - "prompt_tokens": 5910, - "tokens": 5984 } }, { @@ -933,14 +893,6 @@ "metadata": { "model": "claude-sonnet-4-5-20250929", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 109, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 353, - "prompt_cached_tokens": 24171, - "prompt_tokens": 24532, - "tokens": 24641 } } ], @@ -963,6 +915,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 368, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 353, + "prompt_cached_tokens": 48342, + "prompt_tokens": 48713, + "tokens": 49081 } } ], @@ -1021,12 +981,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 2374, - "prompt_cached_tokens": 21870, - "prompt_tokens": 24254 } }, { @@ -1104,12 +1058,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 231, - "prompt_cached_tokens": 24244, - "prompt_tokens": 24483 } } ], @@ -1128,6 +1076,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 267, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2605, + "prompt_cached_tokens": 46114, + "prompt_tokens": 48737, + "tokens": 49004 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt index 17c40ead5..6b91f0755 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt @@ -192,6 +192,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 148, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 488, + │ "prompt_cached_tokens": 24110, + │ "prompt_tokens": 24608, + │ "tokens": 24756 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -228,14 +236,6 @@ span_tree: │ │ "model": "claude-haiku-4-5-20251001", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 475, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 0, - │ │ "prompt_cached_tokens": 24110, - │ │ "prompt_tokens": 24120, - │ │ "tokens": 24595 - │ │ } │ └── anthropic.messages.create [llm] │ input: [ │ { @@ -281,14 +281,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 148, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 488, - │ "prompt_cached_tokens": 24110, - │ "prompt_tokens": 24608, - │ "tokens": 24756 - │ } ├── claude-agent-subagent-operation │ metadata: { │ "operation": "subagent", @@ -314,6 +306,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 294, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 331, + │ "prompt_cached_tokens": 48542, + │ "prompt_tokens": 48891, + │ "tokens": 49185 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -355,14 +355,6 @@ span_tree: │ │ "model": "claude-haiku-4-5-20251001", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 234, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 0, - │ │ "prompt_cached_tokens": 24271, - │ │ "prompt_tokens": 24281, - │ │ "tokens": 24515 - │ │ } │ ├── tool: Agent [tool] │ │ input: { │ │ "description": "Add 15 and 27 using calculator", @@ -486,14 +478,6 @@ span_tree: │ │ │ "model": "claude-haiku-4-5-20251001", │ │ │ "provider": "anthropic" │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 106, - │ │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ │ "prompt_cache_creation_5m_tokens": 2201, - │ │ │ "prompt_cached_tokens": 16794, - │ │ │ "prompt_tokens": 18998, - │ │ │ "tokens": 19104 - │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { │ │ "a": 15, @@ -565,14 +549,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 60, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 331, - │ "prompt_cached_tokens": 24271, - │ "prompt_tokens": 24610, - │ "tokens": 24670 - │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { │ "operation": "subagent-built-in-tool", @@ -599,6 +575,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 368, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 353, + │ "prompt_cached_tokens": 48342, + │ "prompt_tokens": 48713, + │ "tokens": 49081 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -640,14 +624,6 @@ span_tree: │ │ "model": "claude-sonnet-4-5-20250929", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 259, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 0, - │ │ "prompt_cached_tokens": 24171, - │ │ "prompt_tokens": 24181, - │ │ "tokens": 24440 - │ │ } │ ├── tool: Agent [tool] │ │ input: { │ │ "description": "echo greeting", @@ -770,14 +746,6 @@ span_tree: │ │ │ "model": "claude-haiku-4-5-20251001", │ │ │ "provider": "anthropic" │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 74, - │ │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ │ "prompt_cache_creation_5m_tokens": 1359, - │ │ │ "prompt_cached_tokens": 4548, - │ │ │ "prompt_tokens": 5910, - │ │ │ "tokens": 5984 - │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { │ │ "command": "echo hello", @@ -847,14 +815,6 @@ span_tree: │ "model": "claude-sonnet-4-5-20250929", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 109, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 353, - │ "prompt_cached_tokens": 24171, - │ "prompt_tokens": 24532, - │ "tokens": 24641 - │ } └── claude-agent-failure-operation metadata: { "operation": "failure", @@ -877,6 +837,14 @@ span_tree: "permissionMode": "bypassPermissions", "session_id": "" } + metrics: { + "completion_tokens": 267, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2605, + "prompt_cached_tokens": 46114, + "prompt_tokens": 48737, + "tokens": 49004 + } ├── anthropic.messages.create [llm] │ input: [ │ { @@ -918,12 +886,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 2374, - │ "prompt_cached_tokens": 21870, - │ "prompt_tokens": 24254 - │ } ├── tool: calculator/calculator [tool] │ input: { │ "a": 2, @@ -989,9 +951,3 @@ span_tree: "model": "claude-haiku-4-5-20251001", "provider": "anthropic" } - metrics: { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 231, - "prompt_cached_tokens": 24244, - "prompt_tokens": 24483 - } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json index 33a985b81..b6f0987a9 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json @@ -224,14 +224,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 475, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 0, - "prompt_cached_tokens": 24110, - "prompt_tokens": 24120, - "tokens": 24595 } }, { @@ -281,14 +273,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 148, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 488, - "prompt_cached_tokens": 24110, - "prompt_tokens": 24608, - "tokens": 24756 } } ], @@ -323,6 +307,14 @@ "num_turns": 1, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 148, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 488, + "prompt_cached_tokens": 24110, + "prompt_tokens": 24608, + "tokens": 24756 } } ], @@ -381,14 +373,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 234, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 0, - "prompt_cached_tokens": 24271, - "prompt_tokens": 24281, - "tokens": 24515 } }, { @@ -437,14 +421,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 106, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 2201, - "prompt_cached_tokens": 16794, - "prompt_tokens": 18998, - "tokens": 19104 } }, { @@ -617,14 +593,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 60, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 331, - "prompt_cached_tokens": 24271, - "prompt_tokens": 24610, - "tokens": 24670 } } ], @@ -646,6 +614,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 294, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 331, + "prompt_cached_tokens": 48542, + "prompt_tokens": 48891, + "tokens": 49185 } } ], @@ -704,14 +680,6 @@ "metadata": { "model": "claude-sonnet-4-5-20250929", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 259, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 0, - "prompt_cached_tokens": 24171, - "prompt_tokens": 24181, - "tokens": 24440 } }, { @@ -759,14 +727,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 74, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 1359, - "prompt_cached_tokens": 4548, - "prompt_tokens": 5910, - "tokens": 5984 } }, { @@ -933,14 +893,6 @@ "metadata": { "model": "claude-sonnet-4-5-20250929", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 109, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 353, - "prompt_cached_tokens": 24171, - "prompt_tokens": 24532, - "tokens": 24641 } } ], @@ -963,6 +915,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 368, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 353, + "prompt_cached_tokens": 48342, + "prompt_tokens": 48713, + "tokens": 49081 } } ], @@ -1021,12 +981,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 2374, - "prompt_cached_tokens": 21870, - "prompt_tokens": 24254 } }, { @@ -1104,12 +1058,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 231, - "prompt_cached_tokens": 24244, - "prompt_tokens": 24483 } } ], @@ -1128,6 +1076,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 267, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2605, + "prompt_cached_tokens": 46114, + "prompt_tokens": 48737, + "tokens": 49004 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt index 0b6e46dea..dce171e31 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt @@ -192,6 +192,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 148, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 488, + │ "prompt_cached_tokens": 24110, + │ "prompt_tokens": 24608, + │ "tokens": 24756 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -228,14 +236,6 @@ span_tree: │ │ "model": "claude-haiku-4-5-20251001", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 475, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 0, - │ │ "prompt_cached_tokens": 24110, - │ │ "prompt_tokens": 24120, - │ │ "tokens": 24595 - │ │ } │ └── anthropic.messages.create [llm] │ input: [ │ { @@ -281,14 +281,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 148, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 488, - │ "prompt_cached_tokens": 24110, - │ "prompt_tokens": 24608, - │ "tokens": 24756 - │ } ├── claude-agent-subagent-operation │ metadata: { │ "operation": "subagent", @@ -314,6 +306,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 294, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 331, + │ "prompt_cached_tokens": 48542, + │ "prompt_tokens": 48891, + │ "tokens": 49185 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -355,14 +355,6 @@ span_tree: │ │ "model": "claude-haiku-4-5-20251001", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 234, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 0, - │ │ "prompt_cached_tokens": 24271, - │ │ "prompt_tokens": 24281, - │ │ "tokens": 24515 - │ │ } │ ├── tool: Agent [tool] │ │ input: { │ │ "description": "Add 15 and 27 using calculator", @@ -486,14 +478,6 @@ span_tree: │ │ │ "model": "claude-haiku-4-5-20251001", │ │ │ "provider": "anthropic" │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 106, - │ │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ │ "prompt_cache_creation_5m_tokens": 2201, - │ │ │ "prompt_cached_tokens": 16794, - │ │ │ "prompt_tokens": 18998, - │ │ │ "tokens": 19104 - │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { │ │ "a": 15, @@ -565,14 +549,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 60, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 331, - │ "prompt_cached_tokens": 24271, - │ "prompt_tokens": 24610, - │ "tokens": 24670 - │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { │ "operation": "subagent-built-in-tool", @@ -599,6 +575,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 368, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 353, + │ "prompt_cached_tokens": 48342, + │ "prompt_tokens": 48713, + │ "tokens": 49081 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -640,14 +624,6 @@ span_tree: │ │ "model": "claude-sonnet-4-5-20250929", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 259, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 0, - │ │ "prompt_cached_tokens": 24171, - │ │ "prompt_tokens": 24181, - │ │ "tokens": 24440 - │ │ } │ ├── tool: Agent [tool] │ │ input: { │ │ "description": "echo greeting", @@ -770,14 +746,6 @@ span_tree: │ │ │ "model": "claude-haiku-4-5-20251001", │ │ │ "provider": "anthropic" │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 74, - │ │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ │ "prompt_cache_creation_5m_tokens": 1359, - │ │ │ "prompt_cached_tokens": 4548, - │ │ │ "prompt_tokens": 5910, - │ │ │ "tokens": 5984 - │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { │ │ "command": "echo hello", @@ -847,14 +815,6 @@ span_tree: │ "model": "claude-sonnet-4-5-20250929", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 109, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 353, - │ "prompt_cached_tokens": 24171, - │ "prompt_tokens": 24532, - │ "tokens": 24641 - │ } └── claude-agent-failure-operation metadata: { "operation": "failure", @@ -877,6 +837,14 @@ span_tree: "permissionMode": "bypassPermissions", "session_id": "" } + metrics: { + "completion_tokens": 267, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2605, + "prompt_cached_tokens": 46114, + "prompt_tokens": 48737, + "tokens": 49004 + } ├── anthropic.messages.create [llm] │ input: [ │ { @@ -918,12 +886,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 2374, - │ "prompt_cached_tokens": 21870, - │ "prompt_tokens": 24254 - │ } ├── tool: calculator/calculator [tool] │ input: { │ "a": 2, @@ -989,9 +951,3 @@ span_tree: "model": "claude-haiku-4-5-20251001", "provider": "anthropic" } - metrics: { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 231, - "prompt_cached_tokens": 24244, - "prompt_tokens": 24483 - } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json index 1921e7459..48ea5a8cd 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json @@ -223,13 +223,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 171, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 18650, - "prompt_tokens": 18660, - "tokens": 18831 } }, { @@ -279,14 +272,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 190, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 184, - "prompt_cached_tokens": 18650, - "prompt_tokens": 18844, - "tokens": 19034 } } ], @@ -321,6 +306,14 @@ "num_turns": 1, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 190, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 184, + "prompt_cached_tokens": 18650, + "prompt_tokens": 18844, + "tokens": 19034 } } ], @@ -379,13 +372,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 215, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 18811, - "prompt_tokens": 18821, - "tokens": 19036 } }, { @@ -434,13 +420,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 106, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 26759, - "prompt_tokens": 26762, - "tokens": 26868 } }, { @@ -591,14 +570,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 82, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 307, - "prompt_cached_tokens": 18811, - "prompt_tokens": 19126, - "tokens": 19208 } } ], @@ -620,6 +591,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 297, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 19118, + "prompt_cached_tokens": 18811, + "prompt_tokens": 37947, + "tokens": 38244 } } ], @@ -678,13 +657,6 @@ "metadata": { "model": "claude-sonnet-4-5-20250929", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 169, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 18711, - "prompt_tokens": 18721, - "tokens": 18890 } }, { @@ -732,13 +704,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 76, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 17229, - "prompt_tokens": 17232, - "tokens": 17308 } }, { @@ -883,14 +848,6 @@ "metadata": { "model": "claude-sonnet-4-5-20250929", "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 166, - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 250, - "prompt_cached_tokens": 18711, - "prompt_tokens": 18969, - "tokens": 19135 } } ], @@ -913,6 +870,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 335, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18961, + "prompt_cached_tokens": 18711, + "prompt_tokens": 37690, + "tokens": 38025 } } ], @@ -971,12 +936,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 0, - "prompt_cached_tokens": 18784, - "prompt_tokens": 18794 } }, { @@ -1054,12 +1013,6 @@ "metadata": { "model": "claude-haiku-4-5-20251001", "provider": "anthropic" - }, - "metrics": { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 199, - "prompt_cached_tokens": 18784, - "prompt_tokens": 18991 } } ], @@ -1078,6 +1031,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 233, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37785, + "tokens": 38018 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt index aae48e805..56434d2fa 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt @@ -191,6 +191,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 190, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 184, + │ "prompt_cached_tokens": 18650, + │ "prompt_tokens": 18844, + │ "tokens": 19034 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -227,13 +235,6 @@ span_tree: │ │ "model": "claude-haiku-4-5-20251001", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 171, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 18650, - │ │ "prompt_tokens": 18660, - │ │ "tokens": 18831 - │ │ } │ └── anthropic.messages.create [llm] │ input: [ │ { @@ -279,14 +280,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 190, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 184, - │ "prompt_cached_tokens": 18650, - │ "prompt_tokens": 18844, - │ "tokens": 19034 - │ } ├── claude-agent-subagent-operation │ metadata: { │ "operation": "subagent", @@ -312,6 +305,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 297, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 19118, + │ "prompt_cached_tokens": 18811, + │ "prompt_tokens": 37947, + │ "tokens": 38244 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -353,13 +354,6 @@ span_tree: │ │ "model": "claude-haiku-4-5-20251001", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 215, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 18811, - │ │ "prompt_tokens": 18821, - │ │ "tokens": 19036 - │ │ } │ ├── tool: Agent [tool] │ │ input: { │ │ "description": "Add 15 and 27 using calculator", @@ -461,13 +455,6 @@ span_tree: │ │ │ "model": "claude-haiku-4-5-20251001", │ │ │ "provider": "anthropic" │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 106, - │ │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ │ "prompt_cache_creation_5m_tokens": 26759, - │ │ │ "prompt_tokens": 26762, - │ │ │ "tokens": 26868 - │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { │ │ "a": 15, @@ -539,14 +526,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 82, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 307, - │ "prompt_cached_tokens": 18811, - │ "prompt_tokens": 19126, - │ "tokens": 19208 - │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { │ "operation": "subagent-built-in-tool", @@ -573,6 +552,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 335, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 18961, + │ "prompt_cached_tokens": 18711, + │ "prompt_tokens": 37690, + │ "tokens": 38025 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -614,13 +601,6 @@ span_tree: │ │ "model": "claude-sonnet-4-5-20250929", │ │ "provider": "anthropic" │ │ } - │ │ metrics: { - │ │ "completion_tokens": 169, - │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ "prompt_cache_creation_5m_tokens": 18711, - │ │ "prompt_tokens": 18721, - │ │ "tokens": 18890 - │ │ } │ ├── tool: Agent [tool] │ │ input: { │ │ "description": "echo greeting", @@ -721,13 +701,6 @@ span_tree: │ │ │ "model": "claude-haiku-4-5-20251001", │ │ │ "provider": "anthropic" │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 76, - │ │ │ "prompt_cache_creation_1h_tokens": 0, - │ │ │ "prompt_cache_creation_5m_tokens": 17229, - │ │ │ "prompt_tokens": 17232, - │ │ │ "tokens": 17308 - │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { │ │ "command": "echo hello", @@ -797,14 +770,6 @@ span_tree: │ "model": "claude-sonnet-4-5-20250929", │ "provider": "anthropic" │ } - │ metrics: { - │ "completion_tokens": 166, - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 250, - │ "prompt_cached_tokens": 18711, - │ "prompt_tokens": 18969, - │ "tokens": 19135 - │ } └── claude-agent-failure-operation metadata: { "operation": "failure", @@ -827,6 +792,14 @@ span_tree: "permissionMode": "bypassPermissions", "session_id": "" } + metrics: { + "completion_tokens": 233, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37785, + "tokens": 38018 + } ├── anthropic.messages.create [llm] │ input: [ │ { @@ -868,12 +841,6 @@ span_tree: │ "model": "claude-haiku-4-5-20251001", │ "provider": "anthropic" │ } - │ metrics: { - │ "prompt_cache_creation_1h_tokens": 0, - │ "prompt_cache_creation_5m_tokens": 0, - │ "prompt_cached_tokens": 18784, - │ "prompt_tokens": 18794 - │ } ├── tool: calculator/calculator [tool] │ input: { │ "a": 2, @@ -939,9 +906,3 @@ span_tree: "model": "claude-haiku-4-5-20251001", "provider": "anthropic" } - metrics: { - "prompt_cache_creation_1h_tokens": 0, - "prompt_cache_creation_5m_tokens": 199, - "prompt_cached_tokens": 18784, - "prompt_tokens": 18991 - } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts b/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts index 23f90820c..5d22d1c0b 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts @@ -150,7 +150,7 @@ function summarizeSpan( return summary; } -function metricsFromTranscriptUsage(expected: ExpectedUsage): { +function metricsFromExpectedUsage(expected: ExpectedUsage): { completion_tokens: number | undefined; prompt_cache_creation_tokens: number | undefined; prompt_cache_creation_5m_tokens: number | undefined; @@ -169,9 +169,12 @@ function metricsFromTranscriptUsage(expected: ExpectedUsage): { expected.usage?.cache_creation?.ephemeral_1h_input_tokens; const hasCacheCreationBreakdown = cacheCreation5mTokens !== undefined || cacheCreation1hTokens !== undefined; - const effectiveCacheCreationTokens = hasCacheCreationBreakdown - ? (cacheCreation5mTokens ?? 0) + (cacheCreation1hTokens ?? 0) - : aggregateCacheCreationTokens; + const splitCacheCreationTokens = + (cacheCreation5mTokens ?? 0) + (cacheCreation1hTokens ?? 0); + const effectiveCacheCreationTokens = Math.max( + aggregateCacheCreationTokens, + splitCacheCreationTokens, + ); const completionTokens = expected.usage?.output_tokens; const promptTokens = inputTokens + cachedTokens + effectiveCacheCreationTokens; @@ -179,7 +182,9 @@ function metricsFromTranscriptUsage(expected: ExpectedUsage): { return { completion_tokens: completionTokens, prompt_cache_creation_tokens: - !hasCacheCreationBreakdown && aggregateCacheCreationTokens > 0 + (!hasCacheCreationBreakdown || + splitCacheCreationTokens < aggregateCacheCreationTokens) && + aggregateCacheCreationTokens > 0 ? aggregateCacheCreationTokens : undefined, prompt_cache_creation_5m_tokens: cacheCreation5mTokens, @@ -190,31 +195,11 @@ function metricsFromTranscriptUsage(expected: ExpectedUsage): { }; } -function spanUsageMatchesTranscript( - span: CapturedLogEvent | undefined, - expected: ExpectedUsage, -): boolean { - const expectedMetrics = metricsFromTranscriptUsage(expected); - return ( - span?.metrics?.prompt_tokens === expectedMetrics.prompt_tokens && - span.metrics?.completion_tokens === expectedMetrics.completion_tokens && - span.metrics?.tokens === expectedMetrics.tokens && - (span.metrics?.prompt_cached_tokens ?? 0) === - expectedMetrics.prompt_cached_tokens && - span.metrics?.prompt_cache_creation_tokens === - expectedMetrics.prompt_cache_creation_tokens && - span.metrics?.prompt_cache_creation_5m_tokens === - expectedMetrics.prompt_cache_creation_5m_tokens && - span.metrics?.prompt_cache_creation_1h_tokens === - expectedMetrics.prompt_cache_creation_1h_tokens - ); -} - -function expectSpanUsageToMatchTranscript( +function expectSpanUsageToMatch( span: CapturedLogEvent | undefined, expected: ExpectedUsage, ): void { - const expectedMetrics = metricsFromTranscriptUsage(expected); + const expectedMetrics = metricsFromExpectedUsage(expected); expect(span?.metrics).toMatchObject({ ...(expectedMetrics.prompt_cache_creation_tokens !== undefined && { prompt_cache_creation_tokens: @@ -610,9 +595,16 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { expect(expectedUsage?.length).toBeGreaterThan(1); expect(llmSpans).toHaveLength(expectedUsage?.length ?? 0); + expect(task?.metrics?.prompt_tokens).toBeUndefined(); + expect(task?.metrics?.completion_tokens).toBeUndefined(); + expect(task?.metrics?.tokens).toBeUndefined(); + expect(task?.metrics?.prompt_cached_tokens).toBeUndefined(); + expect(task?.metrics?.prompt_cache_creation_tokens).toBeUndefined(); + expect(task?.metrics?.prompt_cache_creation_5m_tokens).toBeUndefined(); + expect(task?.metrics?.prompt_cache_creation_1h_tokens).toBeUndefined(); for (const [index, expected] of (expectedUsage ?? []).entries()) { - expectSpanUsageToMatchTranscript(llmSpans[index], expected); + expectSpanUsageToMatch(llmSpans[index], expected); } }, ); @@ -693,107 +685,53 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { ); test( - "recovers exact usage when partial messages are disabled", + "stores aggregate usage only on tasks without partial messages", testConfig, () => { - const operation = findLatestSpan( + const partialOperation = findLatestSpan( events, - "claude-agent-async-prompt-operation", + "claude-agent-basic-operation", ); - const task = findChildSpans( + const partialTask = findChildSpans( events, "Claude Agent", - operation?.span.id, - ).at(-1); - const expectedUsageSpan = findChildSpans( - events, - "claude-agent-async-prompt-transcript-usage", - operation?.span.id, + partialOperation?.span.id, ).at(-1); - const expectedUsage = expectedUsageSpan?.output as - | ExpectedUsage[] - | undefined; - const llmSpans = findChildSpans( - events, - "anthropic.messages.create", - task?.span.id, - ); - - expect(expectedUsage?.length).toBeGreaterThan(0); - expect(llmSpans).toHaveLength(expectedUsage?.length ?? 0); - for (const [index, expected] of (expectedUsage ?? []).entries()) { - expectSpanUsageToMatchTranscript(llmSpans[index], expected); - } - }, - ); - - test( - "recovers root and separate subagent transcript usage", - testConfig, - () => { - const operation = findLatestSpan( - events, - "claude-agent-subagent-operation", - ); - const taskRoot = findOperationTaskRoot( - events, + const aggregateTasks = [ + "claude-agent-async-prompt-operation", "claude-agent-subagent-operation", - ); - const nestedTask = findSubAgentTaskSpan(events, taskRoot?.span.id); - const expectedUsageSpan = findChildSpans( - events, - "claude-agent-subagent-transcript-usage", - operation?.span.id, - ).at(-1); - const expectedUsage = (expectedUsageSpan?.output ?? - []) as ExpectedUsage[]; - const expectedRootUsage = expectedUsage.filter( - (entry) => entry.parent_tool_use_id === null, - ); - const expectedSubagentUsage = expectedUsage.filter( - (entry) => typeof entry.parent_tool_use_id === "string", - ); - const rootLlmSpans = findChildSpans( - events, - "anthropic.messages.create", - taskRoot?.span.id, - ); - const subagentLlmSpans = findChildSpans( - events, - "anthropic.messages.create", - nestedTask?.span.id, - ); - - expect(expectedRootUsage.length).toBeGreaterThan(0); - expect(expectedSubagentUsage.length).toBeGreaterThan(0); - expect(rootLlmSpans).toHaveLength(expectedRootUsage.length); - expect(subagentLlmSpans.length).toBeGreaterThan(0); - for (const [index, expected] of expectedRootUsage.entries()) { - expectSpanUsageToMatchTranscript(rootLlmSpans[index], expected); - } - for (const span of subagentLlmSpans) { - const hasMatch = expectedSubagentUsage.some((expected) => - spanUsageMatchesTranscript(span, expected), + "claude-agent-subagent-built-in-tool-operation", + "claude-agent-failure-operation", + ].map((operationName) => { + const operation = findLatestSpan(events, operationName); + return findChildSpans(events, "Claude Agent", operation?.span.id).at( + -1, ); - if (!hasMatch) { - throw new Error( - `Subagent span usage did not match transcript: ${JSON.stringify({ expectedSubagentUsage, metrics: span.metrics })}`, - ); - } - } - }, - ); + }); - test( - "does not store aggregate token usage on task spans", - testConfig, - () => { - const taskSpans = findAllSpans(events, "Claude Agent"); - expect(taskSpans.length).toBeGreaterThan(0); - for (const task of taskSpans) { - expect(task.metrics?.prompt_tokens).toBeUndefined(); - expect(task.metrics?.completion_tokens).toBeUndefined(); - expect(task.metrics?.tokens).toBeUndefined(); + expect(partialTask?.metrics?.prompt_tokens).toBeUndefined(); + expect(partialTask?.metrics?.completion_tokens).toBeUndefined(); + expect(partialTask?.metrics?.tokens).toBeUndefined(); + expect(partialTask?.metrics?.prompt_cached_tokens).toBeUndefined(); + expect( + partialTask?.metrics?.prompt_cache_creation_tokens, + ).toBeUndefined(); + expect( + partialTask?.metrics?.prompt_cache_creation_5m_tokens, + ).toBeUndefined(); + expect( + partialTask?.metrics?.prompt_cache_creation_1h_tokens, + ).toBeUndefined(); + for (const task of aggregateTasks) { + expect(task?.metrics).toMatchObject({ + completion_tokens: expect.any(Number), + prompt_tokens: expect.any(Number), + tokens: expect.any(Number), + }); + expect(task?.metrics?.tokens).toBe( + Number(task?.metrics?.prompt_tokens) + + Number(task?.metrics?.completion_tokens), + ); } }, ); @@ -960,7 +898,7 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { } test( - "falls back to prompt usage when the transcript is unavailable", + "omits per-call usage when partial messages are disabled", testConfig, () => { const operation = findLatestSpan( @@ -980,9 +918,13 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { expect(llmSpans.length).toBeGreaterThan(0); for (const llm of llmSpans) { - expect(llm.metrics?.prompt_tokens).toEqual(expect.any(Number)); + expect(llm.metrics?.prompt_tokens).toBeUndefined(); expect(llm.metrics?.completion_tokens).toBeUndefined(); expect(llm.metrics?.tokens).toBeUndefined(); + expect(llm.metrics?.prompt_cached_tokens).toBeUndefined(); + expect(llm.metrics?.prompt_cache_creation_tokens).toBeUndefined(); + expect(llm.metrics?.prompt_cache_creation_5m_tokens).toBeUndefined(); + expect(llm.metrics?.prompt_cache_creation_1h_tokens).toBeUndefined(); } }, ); @@ -1023,11 +965,7 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { async ({ expect }) => { await matchSpanTreeSnapshot( events.filter( - (event) => - event.span.name !== "claude-agent-basic-partial-usage" && - event.span.name !== - "claude-agent-async-prompt-transcript-usage" && - event.span.name !== "claude-agent-subagent-transcript-usage", + (event) => event.span.name !== "claude-agent-basic-partial-usage", ), snapshotPath, { diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs b/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs index 63a0e82a4..836fdf017 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs @@ -1,4 +1,3 @@ -import { readFile } from "node:fs/promises"; import { startSpan, traced, wrapClaudeAgentSDK } from "braintrust"; import { collectAsync, @@ -70,189 +69,6 @@ function assertNoPartialMessages(messages) { } } -function createTranscriptCapture() { - const state = { - rootPath: undefined, - subagentPathByToolUseId: new Map(), - }; - - const captureRootTranscriptPath = async (input) => { - if ( - (input.hook_event_name === "SessionStart" || - input.hook_event_name === "SessionEnd" || - input.hook_event_name === "UserPromptSubmit") && - typeof input.transcript_path === "string" - ) { - state.rootPath = input.transcript_path; - } - return {}; - }; - - return { - hooks: { - SessionStart: [{ hooks: [captureRootTranscriptPath] }], - SessionEnd: [{ hooks: [captureRootTranscriptPath] }], - UserPromptSubmit: [{ hooks: [captureRootTranscriptPath] }], - SubagentStop: [ - { - hooks: [ - async (input, toolUseId) => { - if ( - input.hook_event_name === "SubagentStop" && - typeof toolUseId === "string" && - typeof input.agent_transcript_path === "string" - ) { - state.subagentPathByToolUseId.set( - toolUseId, - input.agent_transcript_path, - ); - } - return {}; - }, - ], - }, - ], - }, - state, - }; -} - -function validTokenCount(value) { - return typeof value === "number" && - Number.isFinite(value) && - Number.isInteger(value) && - value >= 0 - ? value - : undefined; -} - -async function readFinalTranscriptUsage(transcriptPath, parentToolUseId) { - const text = await readFile(transcriptPath, "utf8"); - const usageByMessageId = new Map(); - - for (const line of text.split("\n")) { - if (!line.trim()) { - continue; - } - - let row; - try { - row = JSON.parse(line); - } catch { - continue; - } - - const messageId = row?.type === "assistant" ? row.message?.id : undefined; - const usage = row?.message?.usage; - if (typeof messageId !== "string" || !usage) { - continue; - } - - const inputTokens = validTokenCount(usage.input_tokens); - const outputTokens = validTokenCount(usage.output_tokens); - const cacheReadTokens = validTokenCount(usage.cache_read_input_tokens ?? 0); - const cacheCreationTokens = validTokenCount( - usage.cache_creation_input_tokens ?? 0, - ); - let cacheCreation; - if (usage.cache_creation !== undefined) { - if (!usage.cache_creation || typeof usage.cache_creation !== "object") { - continue; - } - - const cacheCreation5mTokens = validTokenCount( - usage.cache_creation.ephemeral_5m_input_tokens, - ); - const cacheCreation1hTokens = validTokenCount( - usage.cache_creation.ephemeral_1h_input_tokens, - ); - if ( - (usage.cache_creation.ephemeral_5m_input_tokens !== undefined && - cacheCreation5mTokens === undefined) || - (usage.cache_creation.ephemeral_1h_input_tokens !== undefined && - cacheCreation1hTokens === undefined) - ) { - continue; - } - if ( - cacheCreation5mTokens !== undefined || - cacheCreation1hTokens !== undefined - ) { - cacheCreation = { - ...(cacheCreation5mTokens !== undefined && { - ephemeral_5m_input_tokens: cacheCreation5mTokens, - }), - ...(cacheCreation1hTokens !== undefined && { - ephemeral_1h_input_tokens: cacheCreation1hTokens, - }), - }; - } - } - if ( - inputTokens === undefined || - outputTokens === undefined || - cacheReadTokens === undefined || - cacheCreationTokens === undefined - ) { - continue; - } - - usageByMessageId.set(messageId, { - message_id: messageId, - parent_tool_use_id: parentToolUseId, - usage: { - cache_creation_input_tokens: cacheCreationTokens, - cache_read_input_tokens: cacheReadTokens, - input_tokens: inputTokens, - output_tokens: outputTokens, - ...(cacheCreation && { cache_creation: cacheCreation }), - }, - }); - } - - return [...usageByMessageId.values()]; -} - -async function collectCapturedTranscriptUsage(capture, messages) { - if (!capture.state.rootPath) { - throw new Error("User session hook did not receive a transcript path"); - } - - const parentToolUseIdByMessageId = new Map( - messages - .filter( - (message) => - message.type === "assistant" && - typeof message.message?.id === "string", - ) - .map((message) => [ - message.message.id, - message.parent_tool_use_id ?? null, - ]), - ); - const usage = await readFinalTranscriptUsage(capture.state.rootPath, null); - for (const transcriptPath of capture.state.subagentPathByToolUseId.values()) { - const subagentUsage = await readFinalTranscriptUsage(transcriptPath, null); - for (const entry of subagentUsage) { - if (parentToolUseIdByMessageId.has(entry.message_id)) { - entry.parent_tool_use_id = parentToolUseIdByMessageId.get( - entry.message_id, - ); - usage.push(entry); - } - } - } - return usage; -} - -async function logExpectedTranscriptUsage(name, capture, messages) { - const expectedUsageSpan = startSpan({ name }); - expectedUsageSpan.log({ - output: await collectCapturedTranscriptUsage(capture, messages), - }); - expectedUsageSpan.end(); -} - async function collectAsyncAndAssertMessagesUnchanged(records) { const messages = []; const originalMessages = []; @@ -351,7 +167,6 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { "claude-agent-async-prompt-operation", "async-prompt", async () => { - const transcriptCapture = createTranscriptCapture(); const messages = await collectAsyncAndAssertMessagesUnchanged( query({ prompt: (async function* () { @@ -359,7 +174,6 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { yield makePromptMessage("Part 2"); })(), options: { - hooks: transcriptCapture.hooks, includePartialMessages: false, maxTurns: 1, model: CLAUDE_AGENT_MODEL, @@ -368,11 +182,6 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { }), ); assertNoPartialMessages(messages); - await logExpectedTranscriptUsage( - "claude-agent-async-prompt-transcript-usage", - transcriptCapture, - messages, - ); }, ); @@ -380,7 +189,6 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { "claude-agent-subagent-operation", "subagent", async () => { - const transcriptCapture = createTranscriptCapture(); const messages = await collectAsyncAndAssertMessagesUnchanged( query({ prompt: @@ -395,7 +203,6 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { }, }, allowedTools: ["Task"], - hooks: transcriptCapture.hooks, mcpServers: { calculator: calculatorServer, }, @@ -405,11 +212,6 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { }), ); assertNoPartialMessages(messages); - await logExpectedTranscriptUsage( - "claude-agent-subagent-transcript-usage", - transcriptCapture, - messages, - ); }, ); diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index f21f65c81..ffad58aff 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -125,8 +125,8 @@ describe("parseMetricsFromUsage", () => { const result = parseMetricsFromUsageForTest(usage); - // The aggregate is still parsed here; `finalizeAnthropicTokens` is what - // reduces the span to the per-TTL representation. + // The aggregate is still parsed here; `finalizeAnthropicTokens` removes it + // only when the per-TTL representation accounts for the full total. expect(result).toEqual({ prompt_tokens: 100, completion_tokens: 50, diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index a8dfea9b3..36cad5a5c 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -828,8 +828,8 @@ export function parseMetricsFromUsage( // The 5m and 1h cache-write tiers are billed at different rates, so surface // the per-TTL breakdown whenever the response carries it. It is an // alternative representation of `cache_creation_input_tokens` rather than - // additional tokens; `finalizeAnthropicTokens` drops the aggregate so the - // span keeps a single representation. + // additional tokens. `finalizeAnthropicTokens` retains the aggregate as a + // fallback if the breakdown is partial. if (isObject(usage.cache_creation)) { const cacheCreation = usage.cache_creation; for (const [source, target] of [ diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts index 10c656014..f1d446bcd 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts +++ b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts @@ -13,16 +13,6 @@ const streamPatcherMock = vi.hoisted(() => ({ vi.mock("../../isomorph", () => ({ default: { newTracingChannel: vi.fn(), - readFile: vi.fn(), - }, -})); - -vi.mock("../../debug-logger", () => ({ - debugLogger: { - debug: vi.fn(), - error: vi.fn(), - info: vi.fn(), - warn: vi.fn(), }, })); @@ -366,7 +356,7 @@ describe("ClaudeAgentSDKPlugin", () => { expect(true).toBe(true); }); - it("recovers final usage from the transcript without changing query behavior", async () => { + it("keeps transcript hooks untouched and logs usage only on the task span when partial messages are disabled", async () => { const userSessionStart = vi.fn(async (..._args: unknown[]) => ({})); const userSessionStartMatcher = { hooks: [userSessionStart] }; const startEvent = { @@ -386,74 +376,9 @@ describe("ClaudeAgentSDKPlugin", () => { const options = startEvent.arguments[0].options as any; expect(options.includePartialMessages).toBe(false); expect(options.hooks.SessionStart[0]).toBe(userSessionStartMatcher); - expect(options.hooks.SessionStart).toHaveLength(2); - expect(options.hooks.SessionEnd).toHaveLength(1); - expect(options.hooks.UserPromptSubmit).toHaveLength(1); - - const sessionStartInput = { - cwd: "/tmp", - hook_event_name: "SessionStart", - session_id: "session_1", - transcript_path: "/tmp/session.jsonl", - }; - await options.hooks.SessionStart[0].hooks[0]( - sessionStartInput, - undefined, - { signal: new AbortController().signal }, - ); - await options.hooks.SessionStart[1].hooks[0]( - sessionStartInput, - undefined, - { signal: new AbortController().signal }, - ); - expect(userSessionStart).toHaveBeenCalledOnce(); - - vi.mocked(iso.readFile!).mockResolvedValue( - new TextEncoder().encode( - [ - "not json", - JSON.stringify({ - type: "assistant", - message: { - id: "msg_1", - usage: { - cache_creation: { - ephemeral_1h_input_tokens: 0, - ephemeral_5m_input_tokens: 3, - }, - cache_creation_input_tokens: 3, - cache_read_input_tokens: 20, - input_tokens: 10, - output_tokens: 1, - }, - }, - }), - JSON.stringify({ - type: "assistant", - message: { - id: "msg_1", - usage: { - cache_creation: { - ephemeral_1h_input_tokens: 0, - ephemeral_5m_input_tokens: 3, - }, - cache_creation_input_tokens: 3, - cache_read_input_tokens: 20, - input_tokens: 10, - output_tokens: 40, - }, - }, - }), - JSON.stringify({ - type: "assistant", - message: { - id: "msg_1", - usage: { input_tokens: 10, output_tokens: -1 }, - }, - }), - ].join("\n"), - ), - ); + expect(options.hooks.SessionStart).toHaveLength(1); + expect(options.hooks.SessionEnd).toBeUndefined(); + expect(options.hooks.UserPromptSubmit).toBeUndefined(); const stream = { async *[Symbol.asyncIterator]() { @@ -485,16 +410,18 @@ describe("ClaudeAgentSDKPlugin", () => { session_id: "session_1", type: "result", usage: { - cache_creation_input_tokens: 999, - cache_read_input_tokens: 999, - input_tokens: 999, - output_tokens: 999, + cache_creation: { + ephemeral_5m_input_tokens: 3, + }, + cache_creation_input_tokens: 4, + cache_read_input_tokens: 30, + input_tokens: 20, + output_tokens: 40, }, }); await streamPatcherMock.options?.onComplete(); expect(JSON.stringify(assistantMessage)).toBe(originalAssistantMessage); - expect(iso.readFile).toHaveBeenCalledWith("/tmp/session.jsonl"); const llmSpanCallIndex = vi .mocked(startSpan) @@ -508,53 +435,41 @@ describe("ClaudeAgentSDKPlugin", () => { expect(llmSpanCallIndex).toBeGreaterThan(-1); const llmSpan = vi.mocked(startSpan).mock.results[llmSpanCallIndex]?.value; - expect(llmSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ - metrics: { - prompt_cached_tokens: 20, - prompt_tokens: 33, - }, - }), - ); - expect(llmSpan?.log).toHaveBeenLastCalledWith({ + expect( + vi + .mocked(llmSpan!.log) + .mock.calls.some((call: any[]) => call[0].metrics !== undefined), + ).toBe(false); + const taskSpan = vi.mocked(startSpan).mock.results[0]?.value; + expect(taskSpan?.log).toHaveBeenCalledWith({ + metadata: { num_turns: 1, session_id: "session_1" }, metrics: { completion_tokens: 40, - prompt_cache_creation_1h_tokens: 0, prompt_cache_creation_5m_tokens: 3, - prompt_cached_tokens: 20, - prompt_tokens: 33, - tokens: 73, + prompt_cache_creation_tokens: 4, + prompt_cached_tokens: 30, + prompt_tokens: 54, + tokens: 94, }, }); }); - it("omits invalid partial completion usage when the transcript is unavailable", async () => { + it("omits invalid partial completion usage", async () => { const startEvent = { arguments: [ { prompt: "Test", - options: { model: "claude-3-5-sonnet-20241022" }, + options: { + includePartialMessages: true, + model: "claude-3-5-sonnet-20241022", + }, }, ], }; handlers.start(startEvent); - expect( - "includePartialMessages" in startEvent.arguments[0].options, - ).toBe(false); - - const internalSessionStart = (startEvent.arguments[0].options as any) - .hooks.SessionStart[0].hooks[0]; - await internalSessionStart( - { - cwd: "/tmp", - hook_event_name: "SessionStart", - session_id: "session_1", - transcript_path: "/tmp/missing.jsonl", - }, - undefined, - { signal: new AbortController().signal }, + expect(startEvent.arguments[0].options.includePartialMessages).toBe( + true, ); - vi.mocked(iso.readFile!).mockRejectedValue(new Error("missing")); const stream = { async *[Symbol.asyncIterator]() { @@ -623,16 +538,14 @@ describe("ClaudeAgentSDKPlugin", () => { .filter(Boolean); expect(metricLogs).toEqual([ { + prompt_cache_creation_tokens: 3, prompt_cached_tokens: 20, prompt_tokens: 33, }, - { - prompt_cache_creation_tokens: 3, - }, ]); }); - it("keeps the LLM span when individual usage fields are unusable", async () => { + it("keeps the LLM span without usage when partial messages are disabled", async () => { const startEvent = { arguments: [ { @@ -642,7 +555,6 @@ describe("ClaudeAgentSDKPlugin", () => { ], }; handlers.start(startEvent); - vi.mocked(iso.readFile!).mockRejectedValue(new Error("missing")); const stream = { async *[Symbol.asyncIterator]() { @@ -690,7 +602,6 @@ describe("ClaudeAgentSDKPlugin", () => { model: "claude-3-5-sonnet-20241022", provider: "anthropic", }, - metrics: { prompt_tokens: 10 }, output: [ { content: [{ text: "Response", type: "text" }], @@ -699,6 +610,11 @@ describe("ClaudeAgentSDKPlugin", () => { ], }), ); + expect( + vi + .mocked(llmSpan!.log) + .mock.calls.some((call: any[]) => call[0].metrics !== undefined), + ).toBe(false); }); it("layers partial stream usage over the assistant message usage", async () => { @@ -751,7 +667,15 @@ describe("ClaudeAgentSDKPlugin", () => { }, parent_tool_use_id: null, }); - await streamPatcherMock.options?.onChunk?.({ type: "result" }); + await streamPatcherMock.options?.onChunk?.({ + type: "result", + usage: { + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 40, + }, + }); await streamPatcherMock.options?.onComplete(); const llmSpanCallIndex = vi @@ -776,6 +700,12 @@ describe("ClaudeAgentSDKPlugin", () => { }, }), ); + const taskSpan = vi.mocked(startSpan).mock.results[0]?.value; + expect( + vi + .mocked(taskSpan!.log) + .mock.calls.some((call: any[]) => call[0].metrics !== undefined), + ).toBe(false); }); }); diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts index c5ce305d4..9ef41995c 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts +++ b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts @@ -1,8 +1,7 @@ import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; -import iso, { type IsoChannelHandlers } from "../../isomorph"; -import { debugLogger } from "../../debug-logger"; +import type { IsoChannelHandlers } from "../../isomorph"; import { startSpan as startBaseSpan } from "../../logger"; import type { Span } from "../../logger"; import { @@ -52,19 +51,8 @@ type ParentSpanResolver = ( ) => Promise; type LLMSpanResult = { finalMessage: ClaudeConversationMessage | undefined; - span: Span; spanExport: string; }; -type PendingLLMUsage = { - messageId: string; - span: Span; - usage: ClaudeAgentSDKUsage; -}; -type TranscriptUsageState = { - pendingLlmUsageByContextKey: Map; - rootTranscriptPath?: string; - subagentTranscriptPathByToolUseId: Map; -}; type SubAgentDetails = { agentId?: string; agentType?: string; @@ -86,13 +74,6 @@ function llmParentKey(parentToolUseId: string | null): string { return parentToolUseId ?? ROOT_LLM_PARENT_KEY; } -function llmUsageContextKey( - parentToolUseId: string | null, - messageId: string, -): string { - return JSON.stringify([llmParentKey(parentToolUseId), messageId]); -} - function isSubAgentDelegationToolName(toolName: string): boolean { return toolName === "Agent" || toolName === "Task"; } @@ -249,10 +230,7 @@ function tokenCount(value: unknown): number | undefined { * `null` for cache fields they do not populate, and one unusable field must * only cost that metric rather than the whole usage object. */ -function copyUsage( - usage: unknown, - requireInputAndOutput = false, -): ClaudeAgentSDKUsage | undefined { +function copyUsage(usage: unknown): ClaudeAgentSDKUsage | undefined { if (!usage || typeof usage !== "object") { return undefined; } @@ -289,13 +267,6 @@ function copyUsage( } } - if ( - requireInputAndOutput && - (copy.input_tokens === undefined || copy.output_tokens === undefined) - ) { - return undefined; - } - return Object.keys(copy).length > 0 ? copy : undefined; } @@ -319,245 +290,9 @@ function mergeUsage( }; } -function parseTranscriptRowUsage( - line: string, - messageId: string, -): ClaudeAgentSDKUsage | undefined { - let row: unknown; - try { - row = JSON.parse(line); - } catch { - return undefined; - } - - if (!row || typeof row !== "object") { - return undefined; - } - if (getStringProperty(row, "type") !== "assistant") { - return undefined; - } - - const message = Reflect.get(row, "message"); - if (!message || typeof message !== "object") { - return undefined; - } - if (getStringProperty(message, "id") !== messageId) { - return undefined; - } - - return copyUsage(Reflect.get(message, "usage"), true); -} - -function parseTranscriptUsage( - bytes: Uint8Array, - messageIds: Set, -): Map { - const finalUsageByMessageId = new Map(); - if (messageIds.size === 0) { - return finalUsageByMessageId; - } - - const text = new TextDecoder().decode(bytes); - for (const messageId of messageIds) { - // Transcripts for `continue`/`resume` sessions routinely reach tens of - // megabytes, so seek the rows we actually need instead of parsing every - // row in the session on each query completion. - let searchTo = text.length; - while (searchTo >= 0) { - const match = text.lastIndexOf(messageId, searchTo); - if (match < 0) { - break; - } - - const lineStart = text.lastIndexOf("\n", match) + 1; - const lineEnd = text.indexOf("\n", match); - // Transcript rows for one provider request are successive snapshots, not - // separate model calls. The last valid row contains the final usage. - const usage = parseTranscriptRowUsage( - text.slice(lineStart, lineEnd < 0 ? text.length : lineEnd), - messageId, - ); - if (usage) { - finalUsageByMessageId.set(messageId, usage); - break; - } - - searchTo = lineStart - 1; - } - } - - return finalUsageByMessageId; -} - -function recoveredUsage( - pendingUsage: ClaudeAgentSDKUsage, - transcriptUsage: ClaudeAgentSDKUsage, -): ClaudeAgentSDKUsage | undefined { - const inputTokens = tokenCount( - transcriptUsage.input_tokens ?? pendingUsage.input_tokens, - ); - const outputTokens = tokenCount(transcriptUsage.output_tokens); - const cacheReadTokens = tokenCount( - transcriptUsage.cache_read_input_tokens ?? - pendingUsage.cache_read_input_tokens ?? - 0, - ); - const cacheCreationTokens = tokenCount( - transcriptUsage.cache_creation_input_tokens ?? - pendingUsage.cache_creation_input_tokens ?? - 0, - ); - - if ( - inputTokens === undefined || - outputTokens === undefined || - cacheReadTokens === undefined || - cacheCreationTokens === undefined - ) { - return undefined; - } - - const cacheCreation = - transcriptUsage.cache_creation ?? pendingUsage.cache_creation; - - return { - cache_creation_input_tokens: cacheCreationTokens, - cache_read_input_tokens: cacheReadTokens, - input_tokens: inputTokens, - output_tokens: outputTokens, - ...(cacheCreation && { cache_creation: { ...cacheCreation } }), - }; -} - -async function recoverUsageFromTranscript( - state: TranscriptUsageState, -): Promise { - const readFile = iso.readFile; - if (!readFile || state.pendingLlmUsageByContextKey.size === 0) { - return; - } - - const pendingMessageIds = new Set(); - for (const pending of state.pendingLlmUsageByContextKey.values()) { - pendingMessageIds.add(pending.messageId); - } - - const parentToolUseIdsByTranscriptPath = new Map< - string, - Set - >(); - const addTranscriptPath = ( - transcriptPath: string, - parentToolUseId: string | null, - ) => { - const parentToolUseIds = - parentToolUseIdsByTranscriptPath.get(transcriptPath) ?? new Set(); - parentToolUseIds.add(parentToolUseId); - parentToolUseIdsByTranscriptPath.set(transcriptPath, parentToolUseIds); - }; - - if (state.rootTranscriptPath) { - addTranscriptPath(state.rootTranscriptPath, null); - } - for (const [ - toolUseId, - transcriptPath, - ] of state.subagentTranscriptPathByToolUseId) { - addTranscriptPath(transcriptPath, toolUseId); - } - - const transcriptUsages = await Promise.all( - Array.from( - parentToolUseIdsByTranscriptPath, - async ([transcriptPath, parentToolUseIds]) => { - try { - return { - parentToolUseIds, - usageByMessageId: parseTranscriptUsage( - await readFile(transcriptPath), - pendingMessageIds, - ), - }; - } catch (error) { - debugLogger.debug( - "Could not recover Claude Agent SDK transcript usage", - error, - ); - return undefined; - } - }, - ), - ); - - for (const transcriptUsage of transcriptUsages) { - if (!transcriptUsage) { - continue; - } - - for (const parentToolUseId of transcriptUsage.parentToolUseIds) { - for (const [ - messageId, - usageFromTranscript, - ] of transcriptUsage.usageByMessageId) { - const contextKey = llmUsageContextKey(parentToolUseId, messageId); - const pending = state.pendingLlmUsageByContextKey.get(contextKey); - if (!pending) { - continue; - } - - const usage = recoveredUsage(pending.usage, usageFromTranscript); - if (!usage) { - continue; - } - - try { - pending.span.log({ metrics: extractUsage(usage, true) }); - state.pendingLlmUsageByContextKey.delete(contextKey); - } catch (error) { - debugLogger.debug( - "Could not update Claude Agent SDK span with transcript usage", - error, - ); - } - } - } - } -} - -// The CLI appends the final assistant row around the time the query stream -// completes, so give a transcript that is still missing rows a couple of very -// short chances to catch up before giving up on those spans' output tokens. -const TRANSCRIPT_RECOVERY_ATTEMPTS = 3; -const TRANSCRIPT_RECOVERY_RETRY_MS = 25; - -async function recoverUsageWithRetries( - state: TranscriptUsageState, -): Promise { - for (let attempt = 0; attempt < TRANSCRIPT_RECOVERY_ATTEMPTS; attempt++) { - if (attempt > 0) { - await new Promise((resolve) => - setTimeout(resolve, TRANSCRIPT_RECOVERY_RETRY_MS), - ); - } - - await recoverUsageFromTranscript(state); - if (state.pendingLlmUsageByContextKey.size === 0) { - return; - } - if ( - state.rootTranscriptPath === undefined && - state.subagentTranscriptPathByToolUseId.size === 0 - ) { - // No transcript to wait on, so retrying cannot resolve anything. - return; - } - } -} - function extractUsage( usage: ClaudeAgentSDKUsage | undefined, includeOutput: boolean, - omitLegacyCacheCreationMetric = false, ): Record { const metrics: AnthropicTokenMetrics = {}; if (!usage) { @@ -604,20 +339,11 @@ function extractUsage( return {}; } - // `finalizeAnthropicTokens` drops the aggregate cache-creation metric when the - // per-TTL breakdown is present, so the finalized object replaces `metrics` - // rather than being merged back over it. const finalized = finalizeAnthropicTokens(metrics); if (metrics.completion_tokens === undefined) { // A total is only meaningful once both halves are known. delete finalized.tokens; } - if (omitLegacyCacheCreationMetric) { - // Span metrics can only be added, never removed, so hold the aggregate back - // until we know a per-TTL breakdown will not arrive from the transcript. - delete finalized.prompt_cache_creation_tokens; - } - return toNumericMetrics(finalized); } @@ -700,17 +426,16 @@ async function createLLMSpanForMessages( const lastMessage = messages[messages.length - 1]; // Every assistant message is one provider request and must produce one `llm` - // span. Unusable usage costs metrics, never the span or its payloads. + // span. Per-call metrics are attached only when partial messages are enabled; + // otherwise the terminal aggregate belongs exclusively to the task span. if (lastMessage.type !== "assistant") { return undefined; } const model = lastMessage.message?.model || options.model; - const metrics = extractUsage( - usage, - hasFinalOutputUsage, - !hasFinalOutputUsage, - ); + const metrics = options.includePartialMessages + ? extractUsage(usage, hasFinalOutputUsage) + : {}; const input = buildLLMInput(promptMessages, conversationHistory); const outputs = messages .map((m) => @@ -756,7 +481,6 @@ async function createLLMSpanForMessages( return { finalMessage, - span, spanExport, }; } @@ -874,8 +598,6 @@ function createToolTracingHooks( subAgentDetailsByToolUseId: Map, subAgentSpans: Map, endedSubAgentSpans: Set, - transcriptUsageState: TranscriptUsageState, - taskIdToToolUseId: Map, ): { postToolUse: ClaudeAgentSDKHookCallback; postToolUseFailure: ClaudeAgentSDKHookCallback; @@ -1125,14 +847,6 @@ function createToolTracingHooks( toolUseId: toolUseID, }, ); - if (input.agent_transcript_path) { - const parentToolUseId = - taskIdToToolUseId.get(input.agent_id) ?? toolUseID; - transcriptUsageState.subagentTranscriptPathByToolUseId.set( - parentToolUseId, - input.agent_transcript_path, - ); - } const subAgentSpan = subAgentSpans.get(toolUseID); if (!subAgentSpan || endedSubAgentSpans.has(toolUseID)) { return {}; @@ -1174,8 +888,6 @@ function injectTracingHooks( subAgentDetailsByToolUseId: Map, subAgentSpans: Map, endedSubAgentSpans: Set, - transcriptUsageState: TranscriptUsageState, - taskIdToToolUseId: Map, ): ClaudeAgentSDKQueryOptions { const { preToolUse, @@ -1192,26 +904,7 @@ function injectTracingHooks( subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans, - transcriptUsageState, - taskIdToToolUseId, ); - // SessionStart can occur before programmatic hooks are registered in the - // supported SDK versions. UserPromptSubmit carries the same base hook fields - // after registration, while SessionEnd remains a useful final fallback. - const captureRootTranscriptPath: ClaudeAgentSDKHookCallback = async ( - input, - ) => { - if ( - (input.hook_event_name === "SessionStart" || - input.hook_event_name === "SessionEnd" || - input.hook_event_name === "UserPromptSubmit") && - input.agent_id === undefined && - typeof input.transcript_path === "string" - ) { - transcriptUsageState.rootTranscriptPath = input.transcript_path; - } - return {}; - }; const existingHooks = options.hooks ?? {}; @@ -1219,24 +912,6 @@ function injectTracingHooks( ...options, hooks: { ...existingHooks, - SessionStart: [ - ...(existingHooks.SessionStart ?? []), - { - hooks: [captureRootTranscriptPath], - } satisfies ClaudeAgentSDKHookCallbackMatcher, - ], - SessionEnd: [ - ...(existingHooks.SessionEnd ?? []), - { - hooks: [captureRootTranscriptPath], - } satisfies ClaudeAgentSDKHookCallbackMatcher, - ], - UserPromptSubmit: [ - ...(existingHooks.UserPromptSubmit ?? []), - { - hooks: [captureRootTranscriptPath], - } satisfies ClaudeAgentSDKHookCallbackMatcher, - ], PostToolUse: [ ...(existingHooks.PostToolUse ?? []), { hooks: [postToolUse] } satisfies ClaudeAgentSDKHookCallbackMatcher, @@ -1293,7 +968,6 @@ type QueryState = { latestLlmParentBySubAgentToolUse: Map; latestRootLlmParentRef: { value: string | undefined }; toolUseToParent: Map; - transcriptUsageState: TranscriptUsageState; usageByMessageId: Map; localToolContext: ClaudeAgentSDKLocalToolContext; }; @@ -1356,13 +1030,15 @@ async function finalizeCurrentMessageGroup(state: QueryState): Promise { const existingLlmSpan = state.activeLlmSpansByParentToolUse.get(parentKey); const lastMessage = state.currentMessages[state.currentMessages.length - 1]; const messageId = lastMessage?.message?.id; - // Stream events carry the freshest counts, but they can be partial (a - // `message_delta` reports only `output_tokens`), so layer them over the - // assistant message's own usage rather than replacing it. - const usage = mergeUsage( - copyUsage(lastMessage?.message?.usage), - messageId ? state.usageByMessageId.get(messageId) : undefined, - ); + // When partial messages are enabled, stream events carry the freshest + // per-call counts. They can be partial, so layer them over the assistant + // message's usage rather than replacing it. + const usage = state.options.includePartialMessages + ? mergeUsage( + copyUsage(lastMessage?.message?.usage), + messageId ? state.usageByMessageId.get(messageId) : undefined, + ) + : undefined; const hasFinalOutputUsage = messageId !== undefined && state.finalOutputUsageMessageIds.has(messageId); @@ -1392,17 +1068,6 @@ async function finalizeCurrentMessageGroup(state: QueryState): Promise { conversationHistory.push(llmSpanResult.finalMessage); state.finalResults.push(llmSpanResult.finalMessage); } - - if (messageId && !hasFinalOutputUsage && usage) { - state.transcriptUsageState.pendingLlmUsageByContextKey.set( - llmUsageContextKey(parentToolUseId, messageId), - { - messageId, - span: llmSpanResult.span, - usage, - }, - ); - } } // Keep the active LLM parent visible until the finalized exported parent @@ -1834,8 +1499,14 @@ async function handleStreamMessage( if (message.session_id !== undefined) { metadata.session_id = message.session_id; } - if (Object.keys(metadata).length > 0) { - state.span.log({ metadata }); + const metrics = state.options.includePartialMessages + ? {} + : extractUsage(copyUsage(message.usage), true); + if (Object.keys(metadata).length > 0 || Object.keys(metrics).length > 0) { + state.span.log({ + ...(Object.keys(metadata).length > 0 ? { metadata } : {}), + ...(Object.keys(metrics).length > 0 ? { metrics } : {}), + }); } } @@ -1843,39 +1514,6 @@ async function finalizeQuerySpan(state: QueryState): Promise { try { await finalizeCurrentMessageGroup(state); - try { - await recoverUsageWithRetries(state.transcriptUsageState); - } catch (error) { - debugLogger.debug( - "Could not recover Claude Agent SDK transcript usage", - error, - ); - } - for (const pending of state.transcriptUsageState.pendingLlmUsageByContextKey.values()) { - const cacheCreationTokens = pending.usage.cache_creation_input_tokens; - const hasCacheCreationBreakdown = - pending.usage.cache_creation?.ephemeral_5m_input_tokens !== undefined || - pending.usage.cache_creation?.ephemeral_1h_input_tokens !== undefined; - if ( - hasCacheCreationBreakdown || - cacheCreationTokens === undefined || - cacheCreationTokens === 0 - ) { - continue; - } - - try { - pending.span.log({ - metrics: extractAnthropicCacheTokens(0, cacheCreationTokens), - }); - } catch (error) { - debugLogger.debug( - "Could not finalize Claude Agent SDK fallback cache usage", - error, - ); - } - } - state.span.log({ output: state.finalResults.length > 0 @@ -1901,9 +1539,6 @@ async function finalizeQuerySpan(state: QueryState): Promise { state.activePartialMessageIdByParentKey.clear(); state.finalOutputUsageMessageIds.clear(); state.usageByMessageId.clear(); - state.transcriptUsageState.pendingLlmUsageByContextKey.clear(); - state.transcriptUsageState.subagentTranscriptPathByToolUseId.clear(); - state.transcriptUsageState.rootTranscriptPath = undefined; for (const toolSpan of state.activeToolSpans.values()) { toolSpan.end(); @@ -2019,10 +1654,6 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { >(); const promptSourcePriorityByParentKey = new Map(); const localToolContext = createClaudeLocalToolContext(); - const transcriptUsageState: TranscriptUsageState = { - pendingLlmUsageByContextKey: new Map(), - subagentTranscriptPathByToolUseId: new Map(), - }; const { hasLocalToolHandlers, localToolHookNames } = prepareLocalToolHandlersInMcpServers(options.mcpServers); const skipLocalToolHooks = @@ -2084,8 +1715,6 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans, - transcriptUsageState, - taskIdToToolUseId, ); params.options = optionsWithHooks; @@ -2117,7 +1746,6 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { latestLlmParentBySubAgentToolUse, latestRootLlmParentRef, toolUseToParent, - transcriptUsageState, usageByMessageId: new Map(), localToolContext, }); diff --git a/js/src/vendor-sdk-types/claude-agent-sdk.ts b/js/src/vendor-sdk-types/claude-agent-sdk.ts index c558ae2ac..6f33d84bd 100644 --- a/js/src/vendor-sdk-types/claude-agent-sdk.ts +++ b/js/src/vendor-sdk-types/claude-agent-sdk.ts @@ -122,15 +122,6 @@ export interface ClaudeAgentSDKModule { export type ClaudeAgentSDKHookCallback = ( input: - | (BaseHookInput & { - hook_event_name: "SessionStart"; - }) - | (BaseHookInput & { - hook_event_name: "SessionEnd"; - }) - | (BaseHookInput & { - hook_event_name: "UserPromptSubmit"; - }) | (BaseHookInput & { hook_event_name: "PreToolUse"; tool_name: string; diff --git a/js/src/wrappers/anthropic-tokens-util.test.ts b/js/src/wrappers/anthropic-tokens-util.test.ts index 567e82057..b7f1fcc55 100644 --- a/js/src/wrappers/anthropic-tokens-util.test.ts +++ b/js/src/wrappers/anthropic-tokens-util.test.ts @@ -24,7 +24,7 @@ describe("finalizeAnthropicTokens", () => { completion_tokens: 48, prompt_cache_creation_1h_tokens: 0, prompt_cache_creation_5m_tokens: 199, - prompt_cache_creation_tokens: 999, + prompt_cache_creation_tokens: 199, prompt_cached_tokens: 24_243, prompt_tokens: 8, }); @@ -36,7 +36,24 @@ describe("finalizeAnthropicTokens", () => { expect(metrics.prompt_cache_creation_tokens).toBeUndefined(); }); - it("drops the aggregate for callers that keep the returned object", () => { + it("keeps the aggregate when the TTL cache creation breakdown is partial", () => { + expect( + finalizeAnthropicTokens({ + completion_tokens: 48, + prompt_cache_creation_5m_tokens: 199, + prompt_cache_creation_tokens: 999, + prompt_cached_tokens: 24_243, + prompt_tokens: 8, + }), + ).toMatchObject({ + prompt_cache_creation_5m_tokens: 199, + prompt_cache_creation_tokens: 999, + prompt_tokens: 25_250, + tokens: 25_298, + }); + }); + + it("drops a fully represented aggregate for callers that keep the returned object", () => { const raw = { completion_tokens: 48, prompt_cache_creation_5m_tokens: 199, diff --git a/js/src/wrappers/anthropic-tokens-util.ts b/js/src/wrappers/anthropic-tokens-util.ts index 9be227099..ee5efffd3 100644 --- a/js/src/wrappers/anthropic-tokens-util.ts +++ b/js/src/wrappers/anthropic-tokens-util.ts @@ -16,13 +16,13 @@ export interface AnthropicTokenMetrics { } /** - * Rolls cache tokens back into `prompt_tokens`/`tokens` and reduces the - * cache-creation metrics to a single representation. + * Rolls cache tokens back into `prompt_tokens`/`tokens` and removes the + * aggregate cache-creation metric only when the per-TTL breakdown accounts for + * all reported cache-write tokens. * * Callers MUST use the returned object rather than `Object.assign`-ing it back - * over the input: when a per-TTL breakdown is present this drops the aggregate - * `prompt_cache_creation_tokens`, and a merge back over the input would keep - * both representations on the span. + * over the input because finalization may drop + * `prompt_cache_creation_tokens`. */ export function finalizeAnthropicTokens( metrics: AnthropicTokenMetrics, @@ -33,12 +33,12 @@ export function finalizeAnthropicTokens( const splitCacheCreationTokens = (metrics.prompt_cache_creation_5m_tokens || 0) + (metrics.prompt_cache_creation_1h_tokens || 0); - // The split is an alternative representation of the aggregate, not extra - // tokens, and only one of the two is emitted — so `prompt_tokens` is sized - // from whichever representation the span will carry. - const effectiveCacheCreationTokens = hasSplitCacheCreationTokens - ? splitCacheCreationTokens - : metrics.prompt_cache_creation_tokens || 0; + const aggregateCacheCreationTokens = + metrics.prompt_cache_creation_tokens || 0; + const effectiveCacheCreationTokens = Math.max( + aggregateCacheCreationTokens, + splitCacheCreationTokens, + ); const prompt_tokens = (metrics.prompt_tokens || 0) + (metrics.prompt_cached_tokens || 0) + @@ -49,9 +49,11 @@ export function finalizeAnthropicTokens( prompt_tokens, tokens: prompt_tokens + (metrics.completion_tokens || 0), }; - // Anthropic spans must carry exactly one cache-creation representation, and - // the per-TTL breakdown wins whenever the provider reported it. - if (hasSplitCacheCreationTokens) { + // Keep the aggregate as a pricing fallback when the split is partial. + if ( + hasSplitCacheCreationTokens && + splitCacheCreationTokens >= aggregateCacheCreationTokens + ) { delete finalized.prompt_cache_creation_tokens; } return finalized;