diff --git a/.changeset/claude-agent-token-accounting.md b/.changeset/claude-agent-token-accounting.md new file mode 100644 index 000000000..23d727d30 --- /dev/null +++ b/.changeset/claude-agent-token-accounting.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(claude-agent-sdk): Correct per-call token and cost metrics diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-cjs.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-auto-esm.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-cjs.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-auto-esm.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-latest-wrapped.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.json b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.json index e07669995..fa8a7e205 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.json +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -75,7 +76,8 @@ }, "metrics": { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.txt b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.txt index 352ea270d..f24b21676 100644 --- a/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/anthropic-bedrock-instrumentation/__snapshots__/anthropic-bedrock-v0-wrapped.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -66,7 +67,8 @@ span_tree: } metrics: { "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json index cb3b13064..1a4e1fd9c 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -98,7 +99,8 @@ }, "metrics": { "completion_tokens": 6, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 30, "time_to_first_token": 0, @@ -143,7 +145,8 @@ }, "metrics": { "completion_tokens": 7, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 15, "time_to_first_token": 0, @@ -206,7 +209,8 @@ }, "metrics": { "completion_tokens": 27, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "time_to_first_token": 0, @@ -245,7 +249,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -284,7 +289,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -358,7 +364,8 @@ }, "metrics": { "completion_tokens": 26, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -428,7 +435,8 @@ }, "metrics": { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 589, "time_to_first_token": 0, @@ -580,7 +588,8 @@ }, "metrics": { "completion_tokens": 77, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 17662, "server_tool_use_web_fetch_requests": 0, @@ -638,7 +647,8 @@ }, "metrics": { "completion_tokens": 48, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 49, "time_to_first_token": 0, @@ -683,7 +693,8 @@ }, "metrics": { "completion_tokens": 5, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, @@ -722,7 +733,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -761,7 +773,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -835,7 +848,8 @@ }, "metrics": { "completion_tokens": 26, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -910,7 +924,8 @@ }, "metrics": { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 607, "time_to_first_token": 0, @@ -1009,7 +1024,8 @@ }, "metrics": { "completion_tokens": 16, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -1048,7 +1064,8 @@ }, "metrics": { "completion_tokens": 72, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1294, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt index b2830b010..f9d451a62 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -89,7 +90,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 6, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 30, │ "time_to_first_token": 0, @@ -126,7 +128,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 7, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 15, │ "time_to_first_token": 0, @@ -181,7 +184,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 27, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 1389, │ "time_to_first_token": 0, @@ -212,7 +216,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -243,7 +248,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -309,7 +315,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 26, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, @@ -371,7 +378,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 56, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 589, │ "time_to_first_token": 0, @@ -515,7 +523,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 77, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 17662, │ "server_tool_use_web_fetch_requests": 0, @@ -565,7 +574,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 48, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 49, │ "time_to_first_token": 0, @@ -602,7 +612,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 5, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, @@ -633,7 +644,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -664,7 +676,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -730,7 +743,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 26, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, @@ -773,7 +787,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 72, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 1294, │ "time_to_first_token": 0, @@ -831,7 +846,8 @@ span_tree: │ │ } │ │ metrics: { │ │ "completion_tokens": 56, - │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 0, │ │ "prompt_tokens": 607, │ │ "time_to_first_token": 0, @@ -918,7 +934,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 16, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json index cb3b13064..1a4e1fd9c 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -98,7 +99,8 @@ }, "metrics": { "completion_tokens": 6, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 30, "time_to_first_token": 0, @@ -143,7 +145,8 @@ }, "metrics": { "completion_tokens": 7, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 15, "time_to_first_token": 0, @@ -206,7 +209,8 @@ }, "metrics": { "completion_tokens": 27, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "time_to_first_token": 0, @@ -245,7 +249,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -284,7 +289,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -358,7 +364,8 @@ }, "metrics": { "completion_tokens": 26, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -428,7 +435,8 @@ }, "metrics": { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 589, "time_to_first_token": 0, @@ -580,7 +588,8 @@ }, "metrics": { "completion_tokens": 77, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 17662, "server_tool_use_web_fetch_requests": 0, @@ -638,7 +647,8 @@ }, "metrics": { "completion_tokens": 48, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 49, "time_to_first_token": 0, @@ -683,7 +693,8 @@ }, "metrics": { "completion_tokens": 5, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, @@ -722,7 +733,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -761,7 +773,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -835,7 +848,8 @@ }, "metrics": { "completion_tokens": 26, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -910,7 +924,8 @@ }, "metrics": { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 607, "time_to_first_token": 0, @@ -1009,7 +1024,8 @@ }, "metrics": { "completion_tokens": 16, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -1048,7 +1064,8 @@ }, "metrics": { "completion_tokens": 72, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1294, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt index b2830b010..f9d451a62 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -89,7 +90,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 6, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 30, │ "time_to_first_token": 0, @@ -126,7 +128,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 7, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 15, │ "time_to_first_token": 0, @@ -181,7 +184,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 27, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 1389, │ "time_to_first_token": 0, @@ -212,7 +216,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -243,7 +248,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -309,7 +315,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 26, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, @@ -371,7 +378,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 56, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 589, │ "time_to_first_token": 0, @@ -515,7 +523,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 77, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 17662, │ "server_tool_use_web_fetch_requests": 0, @@ -565,7 +574,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 48, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 49, │ "time_to_first_token": 0, @@ -602,7 +612,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 5, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, @@ -633,7 +644,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -664,7 +676,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -730,7 +743,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 26, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, @@ -773,7 +787,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 72, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 1294, │ "time_to_first_token": 0, @@ -831,7 +846,8 @@ span_tree: │ │ } │ │ metrics: { │ │ "completion_tokens": 56, - │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 0, │ │ "prompt_tokens": 607, │ │ "time_to_first_token": 0, @@ -918,7 +934,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 16, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.json index 7f6197d6b..a6dbf0e8c 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.json @@ -36,7 +36,8 @@ }, "metrics": { "completion_tokens": 4, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "time_to_first_token": 0, @@ -98,7 +99,8 @@ }, "metrics": { "completion_tokens": 6, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 30, "time_to_first_token": 0, @@ -143,7 +145,8 @@ }, "metrics": { "completion_tokens": 7, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 15, "time_to_first_token": 0, @@ -206,7 +209,8 @@ }, "metrics": { "completion_tokens": 27, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "time_to_first_token": 0, @@ -245,7 +249,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -284,7 +289,8 @@ }, "metrics": { "completion_tokens": 15, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "time_to_first_token": 0, @@ -358,7 +364,8 @@ }, "metrics": { "completion_tokens": 26, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 687, "time_to_first_token": 0, @@ -428,7 +435,8 @@ }, "metrics": { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 589, "time_to_first_token": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.txt index 2c194b682..e977dda70 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0.span-tree.txt @@ -35,7 +35,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, │ "time_to_first_token": 0, @@ -89,7 +90,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 6, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 30, │ "time_to_first_token": 0, @@ -126,7 +128,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 7, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 15, │ "time_to_first_token": 0, @@ -181,7 +184,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 27, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 1389, │ "time_to_first_token": 0, @@ -212,7 +216,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -243,7 +248,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 15, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 24, │ "time_to_first_token": 0, @@ -309,7 +315,8 @@ span_tree: │ } │ metrics: { │ "completion_tokens": 26, - │ "prompt_cache_creation_tokens": 0, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 687, │ "time_to_first_token": 0, @@ -371,7 +378,8 @@ span_tree: } metrics: { "completion_tokens": 56, - "prompt_cache_creation_tokens": 0, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 589, "time_to_first_token": 0, diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json index 8126cd762..3054e8b65 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.json @@ -52,13 +52,15 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 18783, + "completion_tokens": 174, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18783, "prompt_tokens": 18793, - "tokens": 18796 + "tokens": 18967 } }, { @@ -140,14 +142,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 216, - "prompt_cache_creation_tokens": 18988, + "completion_tokens": 45, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 205, "prompt_cached_tokens": 18783, - "prompt_tokens": 37779, - "tokens": 37995 + "prompt_tokens": 18996, + "tokens": 19041 } } ], @@ -217,13 +221,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 171, - "prompt_cache_creation_tokens": 18650, - "prompt_tokens": 18660, - "tokens": 18831 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -271,14 +270,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 19, - "prompt_cache_creation_tokens": 184, - "prompt_cached_tokens": 18650, - "prompt_tokens": 18844, - "tokens": 18863 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -313,6 +306,14 @@ "num_turns": 1, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 190, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 184, + "prompt_cached_tokens": 18650, + "prompt_tokens": 18844, + "tokens": 19034 } } ], @@ -369,13 +370,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 18811, - "prompt_tokens": 18821, - "tokens": 18824 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -422,13 +418,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 26759, - "prompt_tokens": 26762, - "tokens": 26765 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -577,14 +568,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 291, - "prompt_cache_creation_tokens": 19118, - "prompt_cached_tokens": 18811, - "prompt_tokens": 37937, - "tokens": 38228 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -606,6 +591,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 297, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 19118, + "prompt_cached_tokens": 18811, + "prompt_tokens": 37947, + "tokens": 38244 } } ], @@ -662,13 +655,8 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" - }, - "metrics": { - "completion_tokens": 4, - "prompt_cache_creation_tokens": 18711, - "prompt_tokens": 18721, - "tokens": 18725 + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" } }, { @@ -714,13 +702,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 50, - "prompt_cache_creation_tokens": 17229, - "prompt_tokens": 17232, - "tokens": 17282 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -863,14 +846,8 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" - }, - "metrics": { - "completion_tokens": 281, - "prompt_cache_creation_tokens": 18961, - "prompt_cached_tokens": 18711, - "prompt_tokens": 37680, - "tokens": 37961 + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" } } ], @@ -893,6 +870,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 335, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18961, + "prompt_cached_tokens": 18711, + "prompt_tokens": 37690, + "tokens": 38025 } } ], @@ -949,13 +934,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 8, - "prompt_cached_tokens": 18784, - "prompt_tokens": 18794, - "tokens": 18802 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -1031,14 +1011,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 225, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 37568, - "prompt_tokens": 37775, - "tokens": 38000 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -1057,6 +1031,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 233, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37785, + "tokens": 38018 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt index 581af7a63..775d02457 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-auto-hook.span-tree.txt @@ -64,13 +64,15 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cache_creation_tokens": 18783, + │ │ "completion_tokens": 174, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 18783, │ │ "prompt_tokens": 18793, - │ │ "tokens": 18796 + │ │ "tokens": 18967 │ │ } │ ├── tool: calculator/calculator [tool] │ │ input: { @@ -140,14 +142,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 216, - │ "prompt_cache_creation_tokens": 18988, + │ "completion_tokens": 45, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 205, │ "prompt_cached_tokens": 18783, - │ "prompt_tokens": 37779, - │ "tokens": 37995 + │ "prompt_tokens": 18996, + │ "tokens": 19041 │ } ├── claude-agent-async-prompt-operation │ metadata: { @@ -187,6 +191,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 190, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 184, + │ "prompt_cached_tokens": 18650, + │ "prompt_tokens": 18844, + │ "tokens": 19034 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -220,13 +232,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 171, - │ │ "prompt_cache_creation_tokens": 18650, - │ │ "prompt_tokens": 18660, - │ │ "tokens": 18831 + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ └── anthropic.messages.create [llm] │ input: [ @@ -270,14 +277,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 19, - │ "prompt_cache_creation_tokens": 184, - │ "prompt_cached_tokens": 18650, - │ "prompt_tokens": 18844, - │ "tokens": 18863 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── claude-agent-subagent-operation │ metadata: { @@ -304,6 +305,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 297, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 19118, + │ "prompt_cached_tokens": 18811, + │ "prompt_tokens": 37947, + │ "tokens": 38244 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -342,13 +351,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cache_creation_tokens": 18811, - │ │ "prompt_tokens": 18821, - │ │ "tokens": 18824 + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -448,13 +452,8 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" - │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 3, - │ │ │ "prompt_cache_creation_tokens": 26759, - │ │ │ "prompt_tokens": 26762, - │ │ │ "tokens": 26765 + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { @@ -524,14 +523,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 291, - │ "prompt_cache_creation_tokens": 19118, - │ "prompt_cached_tokens": 18811, - │ "prompt_tokens": 37937, - │ "tokens": 38228 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { @@ -559,6 +552,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 335, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 18961, + │ "prompt_cached_tokens": 18711, + │ "prompt_tokens": 37690, + │ "tokens": 38025 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -597,13 +598,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-sonnet-4-5-20250929" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 4, - │ │ "prompt_cache_creation_tokens": 18711, - │ │ "prompt_tokens": 18721, - │ │ "tokens": 18725 + │ │ "model": "claude-sonnet-4-5-20250929", + │ │ "provider": "anthropic" │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -702,13 +698,8 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" - │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 50, - │ │ │ "prompt_cache_creation_tokens": 17229, - │ │ │ "prompt_tokens": 17232, - │ │ │ "tokens": 17282 + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { @@ -776,14 +767,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-sonnet-4-5-20250929" - │ } - │ metrics: { - │ "completion_tokens": 281, - │ "prompt_cache_creation_tokens": 18961, - │ "prompt_cached_tokens": 18711, - │ "prompt_tokens": 37680, - │ "tokens": 37961 + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic" │ } └── claude-agent-failure-operation metadata: { @@ -807,6 +792,14 @@ span_tree: "permissionMode": "bypassPermissions", "session_id": "" } + metrics: { + "completion_tokens": 233, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37785, + "tokens": 38018 + } ├── anthropic.messages.create [llm] │ input: [ │ { @@ -845,13 +838,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 8, - │ "prompt_cached_tokens": 18784, - │ "prompt_tokens": 18794, - │ "tokens": 18802 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── tool: calculator/calculator [tool] │ input: { @@ -915,12 +903,6 @@ span_tree: } ] metadata: { - "model": "claude-haiku-4-5-20251001" - } - metrics: { - "completion_tokens": 225, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 37568, - "prompt_tokens": 37775, - "tokens": 38000 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json index 79253c91e..bcc7566db 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.json @@ -52,13 +52,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 7, + "completion_tokens": 168, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 24243, "prompt_tokens": 24253, - "tokens": 24260 + "tokens": 24421 } }, { @@ -140,14 +143,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 209, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 48486, - "prompt_tokens": 48693, - "tokens": 48902 + "completion_tokens": 48, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 24243, + "prompt_tokens": 24450, + "tokens": 24498 } } ], @@ -217,13 +222,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 475, - "prompt_cached_tokens": 24110, - "prompt_tokens": 24120, - "tokens": 24595 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -271,14 +271,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 4, - "prompt_cache_creation_tokens": 488, - "prompt_cached_tokens": 24110, - "prompt_tokens": 24608, - "tokens": 24612 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -313,6 +307,14 @@ "num_turns": 1, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 148, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 488, + "prompt_cached_tokens": 24110, + "prompt_tokens": 24608, + "tokens": 24756 } } ], @@ -369,13 +371,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 3, - "prompt_cached_tokens": 24271, - "prompt_tokens": 24281, - "tokens": 24284 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -422,14 +419,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 2201, - "prompt_cached_tokens": 16794, - "prompt_tokens": 18998, - "tokens": 19001 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -600,14 +591,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 288, - "prompt_cache_creation_tokens": 331, - "prompt_cached_tokens": 48542, - "prompt_tokens": 48881, - "tokens": 49169 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -629,6 +614,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 294, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 331, + "prompt_cached_tokens": 48542, + "prompt_tokens": 48891, + "tokens": 49185 } } ], @@ -685,13 +678,8 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" - }, - "metrics": { - "completion_tokens": 3, - "prompt_cached_tokens": 24171, - "prompt_tokens": 24181, - "tokens": 24184 + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" } }, { @@ -737,14 +725,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 57, - "prompt_cache_creation_tokens": 1359, - "prompt_cached_tokens": 4548, - "prompt_tokens": 5910, - "tokens": 5967 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -909,14 +891,8 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" - }, - "metrics": { - "completion_tokens": 308, - "prompt_cache_creation_tokens": 353, - "prompt_cached_tokens": 48342, - "prompt_tokens": 48703, - "tokens": 49011 + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" } } ], @@ -939,6 +915,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 368, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 353, + "prompt_cached_tokens": 48342, + "prompt_tokens": 48713, + "tokens": 49081 } } ], @@ -995,14 +979,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 7, - "prompt_cache_creation_tokens": 2374, - "prompt_cached_tokens": 21870, - "prompt_tokens": 24254, - "tokens": 24261 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -1078,14 +1056,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 260, - "prompt_cache_creation_tokens": 2605, - "prompt_cached_tokens": 46114, - "prompt_tokens": 48727, - "tokens": 48987 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -1104,6 +1076,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 267, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2605, + "prompt_cached_tokens": 46114, + "prompt_tokens": 48737, + "tokens": 49004 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt index 3cd64950c..6b91f0755 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-auto-hook.span-tree.txt @@ -64,13 +64,16 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 7, + │ │ "completion_tokens": 168, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 24243, │ │ "prompt_tokens": 24253, - │ │ "tokens": 24260 + │ │ "tokens": 24421 │ │ } │ ├── tool: calculator/calculator [tool] │ │ input: { @@ -140,14 +143,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 209, - │ "prompt_cache_creation_tokens": 199, - │ "prompt_cached_tokens": 48486, - │ "prompt_tokens": 48693, - │ "tokens": 48902 + │ "completion_tokens": 48, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 199, + │ "prompt_cached_tokens": 24243, + │ "prompt_tokens": 24450, + │ "tokens": 24498 │ } ├── claude-agent-async-prompt-operation │ metadata: { @@ -187,6 +192,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 148, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 488, + │ "prompt_cached_tokens": 24110, + │ "prompt_tokens": 24608, + │ "tokens": 24756 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -220,13 +233,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 475, - │ │ "prompt_cached_tokens": 24110, - │ │ "prompt_tokens": 24120, - │ │ "tokens": 24595 + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ └── anthropic.messages.create [llm] │ input: [ @@ -270,14 +278,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 488, - │ "prompt_cached_tokens": 24110, - │ "prompt_tokens": 24608, - │ "tokens": 24612 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── claude-agent-subagent-operation │ metadata: { @@ -304,6 +306,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 294, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 331, + │ "prompt_cached_tokens": 48542, + │ "prompt_tokens": 48891, + │ "tokens": 49185 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -342,13 +352,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cached_tokens": 24271, - │ │ "prompt_tokens": 24281, - │ │ "tokens": 24284 + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -470,14 +475,8 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" - │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 3, - │ │ │ "prompt_cache_creation_tokens": 2201, - │ │ │ "prompt_cached_tokens": 16794, - │ │ │ "prompt_tokens": 18998, - │ │ │ "tokens": 19001 + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { @@ -547,14 +546,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 288, - │ "prompt_cache_creation_tokens": 331, - │ "prompt_cached_tokens": 48542, - │ "prompt_tokens": 48881, - │ "tokens": 49169 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { @@ -582,6 +575,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 368, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 353, + │ "prompt_cached_tokens": 48342, + │ "prompt_tokens": 48713, + │ "tokens": 49081 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -620,13 +621,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-sonnet-4-5-20250929" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cached_tokens": 24171, - │ │ "prompt_tokens": 24181, - │ │ "tokens": 24184 + │ │ "model": "claude-sonnet-4-5-20250929", + │ │ "provider": "anthropic" │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -747,14 +743,8 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" - │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 57, - │ │ │ "prompt_cache_creation_tokens": 1359, - │ │ │ "prompt_cached_tokens": 4548, - │ │ │ "prompt_tokens": 5910, - │ │ │ "tokens": 5967 + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { @@ -822,14 +812,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-sonnet-4-5-20250929" - │ } - │ metrics: { - │ "completion_tokens": 308, - │ "prompt_cache_creation_tokens": 353, - │ "prompt_cached_tokens": 48342, - │ "prompt_tokens": 48703, - │ "tokens": 49011 + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic" │ } └── claude-agent-failure-operation metadata: { @@ -853,6 +837,14 @@ span_tree: "permissionMode": "bypassPermissions", "session_id": "" } + metrics: { + "completion_tokens": 267, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2605, + "prompt_cached_tokens": 46114, + "prompt_tokens": 48737, + "tokens": 49004 + } ├── anthropic.messages.create [llm] │ input: [ │ { @@ -891,14 +883,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 7, - │ "prompt_cache_creation_tokens": 2374, - │ "prompt_cached_tokens": 21870, - │ "prompt_tokens": 24254, - │ "tokens": 24261 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── tool: calculator/calculator [tool] │ input: { @@ -962,12 +948,6 @@ span_tree: } ] metadata: { - "model": "claude-haiku-4-5-20251001" - } - metrics: { - "completion_tokens": 260, - "prompt_cache_creation_tokens": 2605, - "prompt_cached_tokens": 46114, - "prompt_tokens": 48727, - "tokens": 48987 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json index 3ac00979c..b6f0987a9 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.json @@ -52,13 +52,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 7, + "completion_tokens": 168, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cached_tokens": 24243, "prompt_tokens": 24253, - "tokens": 24260 + "tokens": 24421 } }, { @@ -140,14 +143,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 209, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 48486, - "prompt_tokens": 48693, - "tokens": 48902 + "completion_tokens": 48, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 24243, + "prompt_tokens": 24450, + "tokens": 24498 } } ], @@ -217,13 +222,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 475, - "prompt_cached_tokens": 24110, - "prompt_tokens": 24120, - "tokens": 24595 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -271,14 +271,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 4, - "prompt_cache_creation_tokens": 488, - "prompt_cached_tokens": 24110, - "prompt_tokens": 24608, - "tokens": 24612 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -313,6 +307,14 @@ "num_turns": 1, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 148, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 488, + "prompt_cached_tokens": 24110, + "prompt_tokens": 24608, + "tokens": 24756 } } ], @@ -369,13 +371,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 3, - "prompt_cached_tokens": 24271, - "prompt_tokens": 24281, - "tokens": 24284 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -422,14 +419,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 2201, - "prompt_cached_tokens": 16794, - "prompt_tokens": 18998, - "tokens": 19001 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -600,14 +591,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 288, - "prompt_cache_creation_tokens": 331, - "prompt_cached_tokens": 48542, - "prompt_tokens": 48881, - "tokens": 49169 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -629,6 +614,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 294, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 331, + "prompt_cached_tokens": 48542, + "prompt_tokens": 48891, + "tokens": 49185 } } ], @@ -685,13 +678,8 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" - }, - "metrics": { - "completion_tokens": 3, - "prompt_cached_tokens": 24171, - "prompt_tokens": 24181, - "tokens": 24184 + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" } }, { @@ -737,14 +725,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 57, - "prompt_cache_creation_tokens": 1359, - "prompt_cached_tokens": 4548, - "prompt_tokens": 5910, - "tokens": 5967 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -909,14 +891,8 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" - }, - "metrics": { - "completion_tokens": 308, - "prompt_cache_creation_tokens": 353, - "prompt_cached_tokens": 48342, - "prompt_tokens": 48703, - "tokens": 49011 + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" } } ], @@ -939,6 +915,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 368, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 353, + "prompt_cached_tokens": 48342, + "prompt_tokens": 48713, + "tokens": 49081 } } ], @@ -995,14 +979,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 7, - "prompt_cache_creation_tokens": 2374, - "prompt_cached_tokens": 21870, - "prompt_tokens": 24254, - "tokens": 24261 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -1078,14 +1056,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 260, - "prompt_cache_creation_tokens": 2605, - "prompt_cached_tokens": 46114, - "prompt_tokens": 48727, - "tokens": 48987 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -1104,6 +1076,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 267, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2605, + "prompt_cached_tokens": 46114, + "prompt_tokens": 48737, + "tokens": 49004 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt index 4fd1913ec..dce171e31 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-latest-wrapped.span-tree.txt @@ -64,13 +64,16 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 7, + │ │ "completion_tokens": 168, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 0, │ │ "prompt_cached_tokens": 24243, │ │ "prompt_tokens": 24253, - │ │ "tokens": 24260 + │ │ "tokens": 24421 │ │ } │ ├── tool: calculator/calculator [tool] │ │ input: { @@ -140,14 +143,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 209, - │ "prompt_cache_creation_tokens": 199, - │ "prompt_cached_tokens": 48486, - │ "prompt_tokens": 48693, - │ "tokens": 48902 + │ "completion_tokens": 48, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 199, + │ "prompt_cached_tokens": 24243, + │ "prompt_tokens": 24450, + │ "tokens": 24498 │ } ├── claude-agent-async-prompt-operation │ metadata: { @@ -187,6 +192,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 148, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 488, + │ "prompt_cached_tokens": 24110, + │ "prompt_tokens": 24608, + │ "tokens": 24756 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -220,13 +233,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 475, - │ │ "prompt_cached_tokens": 24110, - │ │ "prompt_tokens": 24120, - │ │ "tokens": 24595 + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ └── anthropic.messages.create [llm] │ input: [ @@ -270,14 +278,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 4, - │ "prompt_cache_creation_tokens": 488, - │ "prompt_cached_tokens": 24110, - │ "prompt_tokens": 24608, - │ "tokens": 24612 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── claude-agent-subagent-operation │ metadata: { @@ -304,6 +306,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 294, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 331, + │ "prompt_cached_tokens": 48542, + │ "prompt_tokens": 48891, + │ "tokens": 49185 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -342,13 +352,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cached_tokens": 24271, - │ │ "prompt_tokens": 24281, - │ │ "tokens": 24284 + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -470,14 +475,8 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" - │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 3, - │ │ │ "prompt_cache_creation_tokens": 2201, - │ │ │ "prompt_cached_tokens": 16794, - │ │ │ "prompt_tokens": 18998, - │ │ │ "tokens": 19001 + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { @@ -547,14 +546,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 288, - │ "prompt_cache_creation_tokens": 331, - │ "prompt_cached_tokens": 48542, - │ "prompt_tokens": 48881, - │ "tokens": 49169 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { @@ -582,6 +575,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 368, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 353, + │ "prompt_cached_tokens": 48342, + │ "prompt_tokens": 48713, + │ "tokens": 49081 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -620,13 +621,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-sonnet-4-5-20250929" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cached_tokens": 24171, - │ │ "prompt_tokens": 24181, - │ │ "tokens": 24184 + │ │ "model": "claude-sonnet-4-5-20250929", + │ │ "provider": "anthropic" │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -747,14 +743,8 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" - │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 57, - │ │ │ "prompt_cache_creation_tokens": 1359, - │ │ │ "prompt_cached_tokens": 4548, - │ │ │ "prompt_tokens": 5910, - │ │ │ "tokens": 5967 + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { @@ -822,14 +812,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-sonnet-4-5-20250929" - │ } - │ metrics: { - │ "completion_tokens": 308, - │ "prompt_cache_creation_tokens": 353, - │ "prompt_cached_tokens": 48342, - │ "prompt_tokens": 48703, - │ "tokens": 49011 + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic" │ } └── claude-agent-failure-operation metadata: { @@ -853,6 +837,14 @@ span_tree: "permissionMode": "bypassPermissions", "session_id": "" } + metrics: { + "completion_tokens": 267, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 2605, + "prompt_cached_tokens": 46114, + "prompt_tokens": 48737, + "tokens": 49004 + } ├── anthropic.messages.create [llm] │ input: [ │ { @@ -891,14 +883,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 7, - │ "prompt_cache_creation_tokens": 2374, - │ "prompt_cached_tokens": 21870, - │ "prompt_tokens": 24254, - │ "tokens": 24261 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── tool: calculator/calculator [tool] │ input: { @@ -962,12 +948,6 @@ span_tree: } ] metadata: { - "model": "claude-haiku-4-5-20251001" - } - metrics: { - "completion_tokens": 260, - "prompt_cache_creation_tokens": 2605, - "prompt_cached_tokens": 46114, - "prompt_tokens": 48727, - "tokens": 48987 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json index d263e9f02..48ea5a8cd 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.json @@ -52,13 +52,15 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 18783, + "completion_tokens": 174, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18783, "prompt_tokens": 18793, - "tokens": 18796 + "tokens": 18967 } }, { @@ -140,14 +142,16 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" }, "metrics": { - "completion_tokens": 216, - "prompt_cache_creation_tokens": 18988, + "completion_tokens": 45, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 205, "prompt_cached_tokens": 18783, - "prompt_tokens": 37779, - "tokens": 37995 + "prompt_tokens": 18996, + "tokens": 19041 } } ], @@ -217,13 +221,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 171, - "prompt_cache_creation_tokens": 18650, - "prompt_tokens": 18660, - "tokens": 18831 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -271,14 +270,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 19, - "prompt_cache_creation_tokens": 184, - "prompt_cached_tokens": 18650, - "prompt_tokens": 18844, - "tokens": 18863 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -313,6 +306,14 @@ "num_turns": 1, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 190, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 184, + "prompt_cached_tokens": 18650, + "prompt_tokens": 18844, + "tokens": 19034 } } ], @@ -369,13 +370,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 18811, - "prompt_tokens": 18821, - "tokens": 18824 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -422,13 +418,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 3, - "prompt_cache_creation_tokens": 26759, - "prompt_tokens": 26762, - "tokens": 26765 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -577,14 +568,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 291, - "prompt_cache_creation_tokens": 19118, - "prompt_cached_tokens": 18811, - "prompt_tokens": 37937, - "tokens": 38228 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -606,6 +591,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 297, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 19118, + "prompt_cached_tokens": 18811, + "prompt_tokens": 37947, + "tokens": 38244 } } ], @@ -662,13 +655,8 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" - }, - "metrics": { - "completion_tokens": 4, - "prompt_cache_creation_tokens": 18711, - "prompt_tokens": 18721, - "tokens": 18725 + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" } }, { @@ -714,13 +702,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 50, - "prompt_cache_creation_tokens": 17229, - "prompt_tokens": 17232, - "tokens": 17282 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -863,14 +846,8 @@ } ], "metadata": { - "model": "claude-sonnet-4-5-20250929" - }, - "metrics": { - "completion_tokens": 281, - "prompt_cache_creation_tokens": 18961, - "prompt_cached_tokens": 18711, - "prompt_tokens": 37680, - "tokens": 37961 + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic" } } ], @@ -893,6 +870,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 335, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 18961, + "prompt_cached_tokens": 18711, + "prompt_tokens": 37690, + "tokens": 38025 } } ], @@ -949,13 +934,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 8, - "prompt_cached_tokens": 18784, - "prompt_tokens": 18794, - "tokens": 18802 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } }, { @@ -1031,14 +1011,8 @@ } ], "metadata": { - "model": "claude-haiku-4-5-20251001" - }, - "metrics": { - "completion_tokens": 225, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 37568, - "prompt_tokens": 37775, - "tokens": 38000 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } } ], @@ -1057,6 +1031,14 @@ "num_turns": 2, "permissionMode": "bypassPermissions", "session_id": "" + }, + "metrics": { + "completion_tokens": 233, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37785, + "tokens": 38018 } } ], diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt index f2d056531..56434d2fa 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0-wrapped.span-tree.txt @@ -64,13 +64,15 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cache_creation_tokens": 18783, + │ │ "completion_tokens": 174, + │ │ "prompt_cache_creation_1h_tokens": 0, + │ │ "prompt_cache_creation_5m_tokens": 18783, │ │ "prompt_tokens": 18793, - │ │ "tokens": 18796 + │ │ "tokens": 18967 │ │ } │ ├── tool: calculator/calculator [tool] │ │ input: { @@ -140,14 +142,16 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 216, - │ "prompt_cache_creation_tokens": 18988, + │ "completion_tokens": 45, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 205, │ "prompt_cached_tokens": 18783, - │ "prompt_tokens": 37779, - │ "tokens": 37995 + │ "prompt_tokens": 18996, + │ "tokens": 19041 │ } ├── claude-agent-async-prompt-operation │ metadata: { @@ -187,6 +191,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 190, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 184, + │ "prompt_cached_tokens": 18650, + │ "prompt_tokens": 18844, + │ "tokens": 19034 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -220,13 +232,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 171, - │ │ "prompt_cache_creation_tokens": 18650, - │ │ "prompt_tokens": 18660, - │ │ "tokens": 18831 + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ └── anthropic.messages.create [llm] │ input: [ @@ -270,14 +277,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 19, - │ "prompt_cache_creation_tokens": 184, - │ "prompt_cached_tokens": 18650, - │ "prompt_tokens": 18844, - │ "tokens": 18863 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── claude-agent-subagent-operation │ metadata: { @@ -304,6 +305,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 297, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 19118, + │ "prompt_cached_tokens": 18811, + │ "prompt_tokens": 37947, + │ "tokens": 38244 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -342,13 +351,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-haiku-4-5-20251001" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 3, - │ │ "prompt_cache_creation_tokens": 18811, - │ │ "prompt_tokens": 18821, - │ │ "tokens": 18824 + │ │ "model": "claude-haiku-4-5-20251001", + │ │ "provider": "anthropic" │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -448,13 +452,8 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" - │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 3, - │ │ │ "prompt_cache_creation_tokens": 26759, - │ │ │ "prompt_tokens": 26762, - │ │ │ "tokens": 26765 + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ └── tool: calculator/calculator [tool] │ │ input: { @@ -524,14 +523,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 291, - │ "prompt_cache_creation_tokens": 19118, - │ "prompt_cached_tokens": 18811, - │ "prompt_tokens": 37937, - │ "tokens": 38228 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── claude-agent-subagent-built-in-tool-operation │ metadata: { @@ -559,6 +552,14 @@ span_tree: │ "permissionMode": "bypassPermissions", │ "session_id": "" │ } + │ metrics: { + │ "completion_tokens": 335, + │ "prompt_cache_creation_1h_tokens": 0, + │ "prompt_cache_creation_5m_tokens": 18961, + │ "prompt_cached_tokens": 18711, + │ "prompt_tokens": 37690, + │ "tokens": 38025 + │ } │ ├── anthropic.messages.create [llm] │ │ input: [ │ │ { @@ -597,13 +598,8 @@ span_tree: │ │ } │ │ ] │ │ metadata: { - │ │ "model": "claude-sonnet-4-5-20250929" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 4, - │ │ "prompt_cache_creation_tokens": 18711, - │ │ "prompt_tokens": 18721, - │ │ "tokens": 18725 + │ │ "model": "claude-sonnet-4-5-20250929", + │ │ "provider": "anthropic" │ │ } │ ├── tool: Agent [tool] │ │ input: { @@ -702,13 +698,8 @@ span_tree: │ │ │ } │ │ │ ] │ │ │ metadata: { - │ │ │ "model": "claude-haiku-4-5-20251001" - │ │ │ } - │ │ │ metrics: { - │ │ │ "completion_tokens": 50, - │ │ │ "prompt_cache_creation_tokens": 17229, - │ │ │ "prompt_tokens": 17232, - │ │ │ "tokens": 17282 + │ │ │ "model": "claude-haiku-4-5-20251001", + │ │ │ "provider": "anthropic" │ │ │ } │ │ └── tool: Bash [tool] │ │ input: { @@ -776,14 +767,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-sonnet-4-5-20250929" - │ } - │ metrics: { - │ "completion_tokens": 281, - │ "prompt_cache_creation_tokens": 18961, - │ "prompt_cached_tokens": 18711, - │ "prompt_tokens": 37680, - │ "tokens": 37961 + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic" │ } └── claude-agent-failure-operation metadata: { @@ -807,6 +792,14 @@ span_tree: "permissionMode": "bypassPermissions", "session_id": "" } + metrics: { + "completion_tokens": 233, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 199, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37785, + "tokens": 38018 + } ├── anthropic.messages.create [llm] │ input: [ │ { @@ -845,13 +838,8 @@ span_tree: │ } │ ] │ metadata: { - │ "model": "claude-haiku-4-5-20251001" - │ } - │ metrics: { - │ "completion_tokens": 8, - │ "prompt_cached_tokens": 18784, - │ "prompt_tokens": 18794, - │ "tokens": 18802 + │ "model": "claude-haiku-4-5-20251001", + │ "provider": "anthropic" │ } ├── tool: calculator/calculator [tool] │ input: { @@ -915,12 +903,6 @@ span_tree: } ] metadata: { - "model": "claude-haiku-4-5-20251001" - } - metrics: { - "completion_tokens": 225, - "prompt_cache_creation_tokens": 199, - "prompt_cached_tokens": 37568, - "prompt_tokens": 37775, - "tokens": 38000 + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic" } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts b/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts index da76ed62c..e2c72b13e 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts @@ -21,6 +21,21 @@ import { import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; +type ExpectedUsage = { + message_id?: string; + parent_tool_use_id?: string | null; + usage?: { + cache_creation_input_tokens?: number; + cache_creation?: { + ephemeral_5m_input_tokens?: number; + ephemeral_1h_input_tokens?: number; + }; + cache_read_input_tokens?: number; + input_tokens?: number; + output_tokens?: number; + }; +}; + type RunClaudeAgentSDKScenario = (harness: { runNodeScenarioDir: (options: { entry?: string; @@ -85,11 +100,6 @@ const SNAPSHOT_METADATA_KEYS = [ "claude_agent_sdk.tool_use_count", "claude_agent_sdk.total_tokens", ] as const; -const OMITTED_METRIC_KEYS = new Set([ - "prompt_cached_tokens", - "prompt_cache_creation_tokens", -]); - function summarizeSpan( event: CapturedLogEvent | undefined, overrides?: { @@ -105,12 +115,7 @@ function summarizeSpan( const summary = summarizeWrapperContract(event, [ ...SNAPSHOT_METADATA_KEYS, ]) as Record; - const metricKeys = Array.isArray(summary.metric_keys) - ? summary.metric_keys.filter( - (key): key is string => - typeof key === "string" && !OMITTED_METRIC_KEYS.has(key), - ) - : summary.metric_keys; + const metricKeys = summary.metric_keys; const input = event.input as | Array<{ content?: string; message?: { content?: string } }> | undefined; @@ -171,6 +176,84 @@ function summarizeSpan( return summary; } +function metricsFromExpectedUsage(expected: ExpectedUsage): { + completion_tokens: number | undefined; + prompt_cache_creation_tokens: number | undefined; + prompt_cache_creation_5m_tokens: number | undefined; + prompt_cache_creation_1h_tokens: number | undefined; + prompt_cached_tokens: number; + prompt_tokens: number; + tokens: number; +} { + const inputTokens = expected.usage?.input_tokens ?? 0; + const cachedTokens = expected.usage?.cache_read_input_tokens ?? 0; + const aggregateCacheCreationTokens = + expected.usage?.cache_creation_input_tokens ?? 0; + const cacheCreation5mTokens = + expected.usage?.cache_creation?.ephemeral_5m_input_tokens; + const cacheCreation1hTokens = + expected.usage?.cache_creation?.ephemeral_1h_input_tokens; + const hasCacheCreationBreakdown = + cacheCreation5mTokens !== undefined || cacheCreation1hTokens !== undefined; + const splitCacheCreationTokens = + (cacheCreation5mTokens ?? 0) + (cacheCreation1hTokens ?? 0); + const effectiveCacheCreationTokens = Math.max( + aggregateCacheCreationTokens, + splitCacheCreationTokens, + ); + const completionTokens = expected.usage?.output_tokens; + const promptTokens = + inputTokens + cachedTokens + effectiveCacheCreationTokens; + + return { + completion_tokens: completionTokens, + prompt_cache_creation_tokens: + (!hasCacheCreationBreakdown || + splitCacheCreationTokens < aggregateCacheCreationTokens) && + aggregateCacheCreationTokens > 0 + ? aggregateCacheCreationTokens + : undefined, + prompt_cache_creation_5m_tokens: cacheCreation5mTokens, + prompt_cache_creation_1h_tokens: cacheCreation1hTokens, + prompt_cached_tokens: cachedTokens, + prompt_tokens: promptTokens, + tokens: promptTokens + (completionTokens ?? 0), + }; +} + +function expectSpanUsageToMatch( + span: CapturedLogEvent | undefined, + expected: ExpectedUsage, +): void { + const expectedMetrics = metricsFromExpectedUsage(expected); + expect(span?.metrics).toMatchObject({ + ...(expectedMetrics.prompt_cache_creation_tokens !== undefined && { + prompt_cache_creation_tokens: + expectedMetrics.prompt_cache_creation_tokens, + }), + ...(expectedMetrics.prompt_cache_creation_5m_tokens !== undefined && { + prompt_cache_creation_5m_tokens: + expectedMetrics.prompt_cache_creation_5m_tokens, + }), + ...(expectedMetrics.prompt_cache_creation_1h_tokens !== undefined && { + prompt_cache_creation_1h_tokens: + expectedMetrics.prompt_cache_creation_1h_tokens, + }), + ...(expectedMetrics.prompt_cached_tokens > 0 && { + prompt_cached_tokens: expectedMetrics.prompt_cached_tokens, + }), + completion_tokens: expectedMetrics.completion_tokens, + prompt_tokens: expectedMetrics.prompt_tokens, + tokens: expectedMetrics.tokens, + }); + if ( + expectedMetrics.prompt_cache_creation_5m_tokens !== undefined || + expectedMetrics.prompt_cache_creation_1h_tokens !== undefined + ) { + expect(span?.metrics?.prompt_cache_creation_tokens).toBeUndefined(); + } +} + function findToolSpanByOperation( events: CapturedLogEvent[], operation: "add" | "divide" | "multiply" | "subtract", @@ -511,6 +594,49 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); }); + test( + "attributes exact provider stream usage to each basic llm call", + testConfig, + () => { + const operation = findLatestSpan( + events, + "claude-agent-basic-operation", + ); + const task = findChildSpans( + events, + "Claude Agent", + operation?.span.id, + ).at(-1); + const expectedUsageSpan = findChildSpans( + events, + "claude-agent-basic-partial-usage", + operation?.span.id, + ).at(-1); + const expectedUsage = expectedUsageSpan?.output as + | ExpectedUsage[] + | undefined; + const llmSpans = findChildSpans( + events, + "anthropic.messages.create", + task?.span.id, + ); + + expect(expectedUsage?.length).toBeGreaterThan(1); + expect(llmSpans).toHaveLength(expectedUsage?.length ?? 0); + expect(task?.metrics?.prompt_tokens).toBeUndefined(); + expect(task?.metrics?.completion_tokens).toBeUndefined(); + expect(task?.metrics?.tokens).toBeUndefined(); + expect(task?.metrics?.prompt_cached_tokens).toBeUndefined(); + expect(task?.metrics?.prompt_cache_creation_tokens).toBeUndefined(); + expect(task?.metrics?.prompt_cache_creation_5m_tokens).toBeUndefined(); + expect(task?.metrics?.prompt_cache_creation_1h_tokens).toBeUndefined(); + + for (const [index, expected] of (expectedUsage ?? []).entries()) { + expectSpanUsageToMatch(llmSpans[index], expected); + } + }, + ); + if (options.assertLocalToolHandlerParenting) { test( "nests local tool handler spans under tool spans", @@ -586,6 +712,58 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { }, ); + test( + "stores aggregate usage only on tasks without partial messages", + testConfig, + () => { + const partialOperation = findLatestSpan( + events, + "claude-agent-basic-operation", + ); + const partialTask = findChildSpans( + events, + "Claude Agent", + partialOperation?.span.id, + ).at(-1); + const aggregateTasks = [ + "claude-agent-async-prompt-operation", + "claude-agent-subagent-operation", + "claude-agent-subagent-built-in-tool-operation", + "claude-agent-failure-operation", + ].map((operationName) => { + const operation = findLatestSpan(events, operationName); + return findChildSpans(events, "Claude Agent", operation?.span.id).at( + -1, + ); + }); + + expect(partialTask?.metrics?.prompt_tokens).toBeUndefined(); + expect(partialTask?.metrics?.completion_tokens).toBeUndefined(); + expect(partialTask?.metrics?.tokens).toBeUndefined(); + expect(partialTask?.metrics?.prompt_cached_tokens).toBeUndefined(); + expect( + partialTask?.metrics?.prompt_cache_creation_tokens, + ).toBeUndefined(); + expect( + partialTask?.metrics?.prompt_cache_creation_5m_tokens, + ).toBeUndefined(); + expect( + partialTask?.metrics?.prompt_cache_creation_1h_tokens, + ).toBeUndefined(); + for (const task of aggregateTasks) { + expect(task?.metrics).toMatchObject({ + completion_tokens: expect.any(Number), + prompt_tokens: expect.any(Number), + tokens: expect.any(Number), + }); + expect(task?.metrics?.tokens).toBe( + Number(task?.metrics?.prompt_tokens) + + Number(task?.metrics?.completion_tokens), + ); + } + }, + ); + test("captures nested subagent task hierarchy", testConfig, () => { const operation = findLatestSpan( events, @@ -766,6 +944,38 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { ); } + test( + "omits per-call usage when partial messages are disabled", + testConfig, + () => { + const operation = findLatestSpan( + events, + "claude-agent-failure-operation", + ); + const task = findChildSpans( + events, + "Claude Agent", + operation?.span.id, + ).at(-1); + const llmSpans = findChildSpans( + events, + "anthropic.messages.create", + task?.span.id, + ); + + expect(llmSpans.length).toBeGreaterThan(0); + for (const llm of llmSpans) { + expect(llm.metrics?.prompt_tokens).toBeUndefined(); + expect(llm.metrics?.completion_tokens).toBeUndefined(); + expect(llm.metrics?.tokens).toBeUndefined(); + expect(llm.metrics?.prompt_cached_tokens).toBeUndefined(); + expect(llm.metrics?.prompt_cache_creation_tokens).toBeUndefined(); + expect(llm.metrics?.prompt_cache_creation_5m_tokens).toBeUndefined(); + expect(llm.metrics?.prompt_cache_creation_1h_tokens).toBeUndefined(); + } + }, + ); + test("captures tool failure details", testConfig, () => { const operation = findLatestSpan( events, @@ -800,9 +1010,15 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { "matches the shared span tree snapshot", testConfig, async ({ expect }) => { - await matchSpanTreeSnapshot(events, snapshotPath, { - snapshotExpect: expect, - }); + await matchSpanTreeSnapshot( + events.filter( + (event) => event.span.name !== "claude-agent-basic-partial-usage", + ), + snapshotPath, + { + snapshotExpect: expect, + }, + ); }, ); }); diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/cassette-filter.mjs b/e2e/scenarios/claude-agent-sdk-instrumentation/cassette-filter.mjs index 7890b9d5c..2bab107db 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/cassette-filter.mjs +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/cassette-filter.mjs @@ -5,10 +5,24 @@ export const filter = [ { normalizeRequest(req) { const url = new URL(req.url); - if (req.method === "POST" && url.hostname === "api.anthropic.com") { - return { ...req, body: { kind: "empty" } }; + if (req.method !== "POST" || url.hostname !== "api.anthropic.com") { + return req; } - return req; + + // Claude can send a title request at the same time as a model request. + // Give each request class a separate cassette sequence. Large request + // bodies contain volatile Claude system data, so match them by order. + const requestClass = + req.body?.kind === "json" && req.body.value?.output_config + ? "title" + : "model"; + url.pathname = `${url.pathname}/__cassette_${requestClass}`; + + return { + ...req, + body: { kind: "empty" }, + url: url.toString(), + }; }, }, ]; diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs b/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs index 680fa4fa7..d99cbfa52 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs @@ -1,4 +1,4 @@ -import { traced, wrapClaudeAgentSDK } from "braintrust"; +import { startSpan, traced, wrapClaudeAgentSDK } from "braintrust"; import { collectAsync, runOperation, @@ -22,6 +22,71 @@ function makePromptMessage(content) { }; } +function collectFinalPartialUsage(messages) { + const activeMessageByParent = new Map(); + const usageByMessageId = new Map(); + + for (const message of messages) { + if (message.type !== "stream_event") { + continue; + } + + const parentKey = message.parent_tool_use_id ?? "__root__"; + const event = message.event; + if (event?.type === "message_start" && event.message?.id) { + activeMessageByParent.set(parentKey, event.message.id); + usageByMessageId.set(event.message.id, { + message_id: event.message.id, + parent_tool_use_id: message.parent_tool_use_id ?? null, + usage: { ...event.message.usage }, + }); + continue; + } + + const messageId = activeMessageByParent.get(parentKey); + if (!messageId) { + continue; + } + + if (event?.type === "message_delta") { + const captured = usageByMessageId.get(messageId); + if (captured && event.usage) { + Object.assign(captured.usage, event.usage); + } + } else if (event?.type === "message_stop") { + activeMessageByParent.delete(parentKey); + } + } + + return [...usageByMessageId.values()]; +} + +function assertNoPartialMessages(messages) { + if (messages.some((message) => message.type === "stream_event")) { + throw new Error( + "Braintrust exposed internally enabled Claude Agent SDK partial messages", + ); + } +} + +async function collectAsyncAndAssertMessagesUnchanged(records, onMessage) { + const messages = []; + const originalMessages = []; + for await (const message of records) { + messages.push(message); + originalMessages.push(JSON.stringify(message)); + onMessage?.(message); + } + + for (const [index, message] of messages.entries()) { + if (JSON.stringify(message) !== originalMessages[index]) { + throw new Error("Braintrust mutated a Claude Agent SDK message"); + } + } + + return messages; +} + async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { const instrumentedSDK = decorateSDK ? decorateSDK(sdk) : sdk; const { createSdkMcpServer, query, tool } = instrumentedSDK; @@ -75,11 +140,12 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { await runTracedScenario({ callback: async () => { await runOperation("claude-agent-basic-operation", "basic", async () => { - await collectAsync( + const messages = await collectAsyncAndAssertMessagesUnchanged( query({ prompt: "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", options: { + includePartialMessages: true, mcpServers: { calculator: calculatorServer, }, @@ -88,25 +154,34 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { }, }), ); + const expectedUsageSpan = startSpan({ + name: "claude-agent-basic-partial-usage", + }); + expectedUsageSpan.log({ + output: collectFinalPartialUsage(messages), + }); + expectedUsageSpan.end(); }); await runOperation( "claude-agent-async-prompt-operation", "async-prompt", async () => { - await collectAsync( + const messages = await collectAsyncAndAssertMessagesUnchanged( query({ prompt: (async function* () { yield makePromptMessage("Part 1"); yield makePromptMessage("Part 2"); })(), options: { + includePartialMessages: false, maxTurns: 1, model: CLAUDE_AGENT_MODEL, permissionMode: "bypassPermissions", }, }), ); + assertNoPartialMessages(messages); }, ); @@ -115,61 +190,63 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { "subagent", async () => { const consumedToolUseIds = new Set(); - const result = query({ - prompt: - "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", - options: { - agents: { - "math-expert": { - description: "Math specialist", - model: CLAUDE_AGENT_MODEL, - prompt: - "You are a math expert. Use the calculator tool for calculations. Be concise.", - }, - }, - allowedTools: ["Task"], - hooks: { - PreToolUse: [ - { - hooks: [ - async (input, toolUseId) => { - if ( - input.tool_name === "mcp__calculator__calculator" && - input.tool_input?.operation === "add" && - typeof toolUseId === "string" && - !consumedToolUseIds.has(toolUseId) - ) { - subagentRacedToolUseId = toolUseId; - } - return {}; - }, - ], + const messages = await collectAsyncAndAssertMessagesUnchanged( + query({ + prompt: + "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + options: { + agents: { + "math-expert": { + description: "Math specialist", + model: CLAUDE_AGENT_MODEL, + prompt: + "You are a math expert. Use the calculator tool for calculations. Be concise.", }, - ], - }, - mcpServers: { - calculator: calculatorServer, + }, + allowedTools: ["Task"], + hooks: { + PreToolUse: [ + { + hooks: [ + async (input, toolUseId) => { + if ( + input.tool_name === "mcp__calculator__calculator" && + input.tool_input?.operation === "add" && + typeof toolUseId === "string" && + !consumedToolUseIds.has(toolUseId) + ) { + subagentRacedToolUseId = toolUseId; + } + return {}; + }, + ], + }, + ], + }, + mcpServers: { + calculator: calculatorServer, + }, + model: CLAUDE_AGENT_MODEL, + permissionMode: "bypassPermissions", }, - model: CLAUDE_AGENT_MODEL, - permissionMode: "bypassPermissions", - }, - }); - - for await (const record of result) { - if ( - record.type === "assistant" && - Array.isArray(record.message?.content) - ) { - for (const block of record.message.content) { - if ( - block?.type === "tool_use" && - typeof block.id === "string" - ) { - consumedToolUseIds.add(block.id); + }), + (record) => { + if ( + record.type === "assistant" && + Array.isArray(record.message?.content) + ) { + for (const block of record.message.content) { + if ( + block?.type === "tool_use" && + typeof block.id === "string" + ) { + consumedToolUseIds.add(block.id); + } } } - } - } + }, + ); + assertNoPartialMessages(messages); }, ); @@ -204,7 +281,7 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { "claude-agent-failure-operation", "failure", async () => { - await collectAsync( + const messages = await collectAsyncAndAssertMessagesUnchanged( query({ prompt: "Use the calculator tool to divide 2 by 0. Do not recover from the error.", @@ -214,9 +291,11 @@ async function runClaudeAgentSDKScenario({ decorateSDK, sdk }) { }, model: CLAUDE_AGENT_MODEL, permissionMode: "bypassPermissions", + persistSession: false, }, }), ); + assertNoPartialMessages(messages); }, ); }, diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index c8aa0b955..ffad58aff 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -111,6 +111,54 @@ describe("parseMetricsFromUsage", () => { }); }); + it("should surface the per-TTL cache creation breakdown", () => { + const usage = { + input_tokens: 100, + output_tokens: 50, + cache_read_input_tokens: 25, + cache_creation_input_tokens: 30, + cache_creation: { + ephemeral_1h_input_tokens: 10, + ephemeral_5m_input_tokens: 20, + }, + }; + + const result = parseMetricsFromUsageForTest(usage); + + // The aggregate is still parsed here; `finalizeAnthropicTokens` removes it + // only when the per-TTL representation accounts for the full total. + expect(result).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_cached_tokens: 25, + prompt_cache_creation_tokens: 30, + prompt_cache_creation_1h_tokens: 10, + prompt_cache_creation_5m_tokens: 20, + }); + }); + + it("should ignore a null or partial cache creation breakdown", () => { + expect( + parseMetricsFromUsageForTest({ + input_tokens: 100, + output_tokens: 50, + cache_creation: null, + }), + ).toEqual({ prompt_tokens: 100, completion_tokens: 50 }); + + expect( + parseMetricsFromUsageForTest({ + input_tokens: 100, + output_tokens: 50, + cache_creation: { ephemeral_5m_input_tokens: 20 }, + }), + ).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_cache_creation_5m_tokens: 20, + }); + }); + it("should ignore non-number token values", () => { const usage = { input_tokens: "not a number", diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index 261223e91..36cad5a5c 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -825,6 +825,24 @@ export function parseMetricsFromUsage( saveIfExistsTo("cache_read_input_tokens", "prompt_cached_tokens"); saveIfExistsTo("cache_creation_input_tokens", "prompt_cache_creation_tokens"); + // The 5m and 1h cache-write tiers are billed at different rates, so surface + // the per-TTL breakdown whenever the response carries it. It is an + // alternative representation of `cache_creation_input_tokens` rather than + // additional tokens. `finalizeAnthropicTokens` retains the aggregate as a + // fallback if the breakdown is partial. + if (isObject(usage.cache_creation)) { + const cacheCreation = usage.cache_creation; + for (const [source, target] of [ + ["ephemeral_5m_input_tokens", "prompt_cache_creation_5m_tokens"], + ["ephemeral_1h_input_tokens", "prompt_cache_creation_1h_tokens"], + ] as const) { + const value = cacheCreation[source]; + if (typeof value === "number") { + metrics[target] = value; + } + } + } + if (isObject(usage.server_tool_use)) { for (const [name, value] of Object.entries(usage.server_tool_use)) { if (typeof value === "number") { diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts index 6eef8c572..f1d446bcd 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts +++ b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts @@ -1,12 +1,35 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; // Mock iso's newTracingChannel - must be before any imports that use it +const streamPatcherMock = vi.hoisted(() => ({ + options: undefined as + | { + onChunk?: (chunk: unknown) => void | Promise; + onComplete: () => void | Promise; + } + | undefined, +})); + vi.mock("../../isomorph", () => ({ default: { newTracingChannel: vi.fn(), }, })); +vi.mock("../core/stream-patcher", () => ({ + isAsyncIterable: vi.fn( + (val: unknown) => + val !== null && + typeof val === "object" && + Symbol.asyncIterator in val && + typeof (val as any)[Symbol.asyncIterator] === "function", + ), + patchStreamIfNeeded: vi.fn((stream, options) => { + streamPatcherMock.options = options; + return stream; + }), +})); + import { ClaudeAgentSDKPlugin } from "./claude-agent-sdk-plugin"; import iso from "../../isomorph"; import { startSpan } from "../../logger"; @@ -45,16 +68,9 @@ vi.mock("../../wrappers/attachment-utils", () => ({ processInputAttachments: vi.fn((input) => input), })); -vi.mock("../../wrappers/anthropic-tokens-util", () => ({ - extractAnthropicCacheTokens: vi.fn((read, creation) => ({ - prompt_cache_read_tokens: read, - prompt_cache_creation_tokens: creation, - })), - finalizeAnthropicTokens: vi.fn((metrics) => ({ - ...metrics, - tokens: (metrics.prompt_tokens || 0) + (metrics.completion_tokens || 0), - })), -})); +// `anthropic-tokens-util` is pure and owns the cache-creation representation +// rules, so these tests run the real implementation rather than a stand-in that +// could drift from it. vi.mock("../core", async (importOriginal) => { const actual = await importOriginal(); @@ -109,6 +125,7 @@ describe("ClaudeAgentSDKPlugin", () => { let mockUnsubscribe: any; beforeEach(() => { + streamPatcherMock.options = undefined; mockUnsubscribe = vi.fn(); mockChannel = { subscribe: vi.fn(), @@ -338,6 +355,358 @@ describe("ClaudeAgentSDKPlugin", () => { // Should not throw expect(true).toBe(true); }); + + it("keeps transcript hooks untouched and logs usage only on the task span when partial messages are disabled", async () => { + const userSessionStart = vi.fn(async (..._args: unknown[]) => ({})); + const userSessionStartMatcher = { hooks: [userSessionStart] }; + const startEvent = { + arguments: [ + { + prompt: "Test", + options: { + hooks: { SessionStart: [userSessionStartMatcher] }, + includePartialMessages: false, + model: "claude-3-5-sonnet-20241022", + }, + }, + ], + }; + handlers.start(startEvent); + + const options = startEvent.arguments[0].options as any; + expect(options.includePartialMessages).toBe(false); + expect(options.hooks.SessionStart[0]).toBe(userSessionStartMatcher); + expect(options.hooks.SessionStart).toHaveLength(1); + expect(options.hooks.SessionEnd).toBeUndefined(); + expect(options.hooks.UserPromptSubmit).toBeUndefined(); + + const stream = { + async *[Symbol.asyncIterator]() { + // The patcher is mocked; messages are delivered below. + }, + }; + handlers.end(Object.assign(startEvent, { result: stream })); + + const assistantMessage = { + type: "assistant", + message: { + content: [{ text: "Response", type: "text" }], + id: "msg_1", + model: "claude-3-5-sonnet-20241022", + role: "assistant", + usage: { + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 1, + }, + }, + parent_tool_use_id: null, + }; + const originalAssistantMessage = JSON.stringify(assistantMessage); + await streamPatcherMock.options?.onChunk?.(assistantMessage); + await streamPatcherMock.options?.onChunk?.({ + num_turns: 1, + session_id: "session_1", + type: "result", + usage: { + cache_creation: { + ephemeral_5m_input_tokens: 3, + }, + cache_creation_input_tokens: 4, + cache_read_input_tokens: 30, + input_tokens: 20, + output_tokens: 40, + }, + }); + await streamPatcherMock.options?.onComplete(); + + expect(JSON.stringify(assistantMessage)).toBe(originalAssistantMessage); + + const llmSpanCallIndex = vi + .mocked(startSpan) + .mock.calls.findIndex( + ([args]) => + args && + typeof args === "object" && + "name" in args && + args.name === "anthropic.messages.create", + ); + expect(llmSpanCallIndex).toBeGreaterThan(-1); + const llmSpan = + vi.mocked(startSpan).mock.results[llmSpanCallIndex]?.value; + expect( + vi + .mocked(llmSpan!.log) + .mock.calls.some((call: any[]) => call[0].metrics !== undefined), + ).toBe(false); + const taskSpan = vi.mocked(startSpan).mock.results[0]?.value; + expect(taskSpan?.log).toHaveBeenCalledWith({ + metadata: { num_turns: 1, session_id: "session_1" }, + metrics: { + completion_tokens: 40, + prompt_cache_creation_5m_tokens: 3, + prompt_cache_creation_tokens: 4, + prompt_cached_tokens: 30, + prompt_tokens: 54, + tokens: 94, + }, + }); + }); + + it("omits invalid partial completion usage", async () => { + const startEvent = { + arguments: [ + { + prompt: "Test", + options: { + includePartialMessages: true, + model: "claude-3-5-sonnet-20241022", + }, + }, + ], + }; + handlers.start(startEvent); + expect(startEvent.arguments[0].options.includePartialMessages).toBe( + true, + ); + + const stream = { + async *[Symbol.asyncIterator]() { + // The patcher is mocked; messages are delivered below. + }, + }; + handlers.end(Object.assign(startEvent, { result: stream })); + await streamPatcherMock.options?.onChunk?.({ + type: "stream_event", + event: { + type: "message_start", + message: { + id: "msg_missing", + usage: { + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 1, + }, + }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ + type: "stream_event", + event: { + type: "message_delta", + usage: { output_tokens: -1 }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ + type: "assistant", + message: { + content: [{ text: "Response", type: "text" }], + id: "msg_missing", + role: "assistant", + usage: { + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 1, + }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ type: "result" }); + await expect( + streamPatcherMock.options?.onComplete(), + ).resolves.toBeUndefined(); + + const llmSpanCallIndex = vi + .mocked(startSpan) + .mock.calls.findIndex( + ([args]) => + args && + typeof args === "object" && + "name" in args && + args.name === "anthropic.messages.create", + ); + const llmSpan = + vi.mocked(startSpan).mock.results[llmSpanCallIndex]?.value; + const metricLogs = vi + .mocked(llmSpan!.log) + .mock.calls.map((call: any[]) => call[0].metrics) + .filter(Boolean); + expect(metricLogs).toEqual([ + { + prompt_cache_creation_tokens: 3, + prompt_cached_tokens: 20, + prompt_tokens: 33, + }, + ]); + }); + + it("keeps the LLM span without usage when partial messages are disabled", async () => { + const startEvent = { + arguments: [ + { + prompt: "Test", + options: { model: "claude-3-5-sonnet-20241022" }, + }, + ], + }; + handlers.start(startEvent); + + const stream = { + async *[Symbol.asyncIterator]() { + // The patcher is mocked; messages are delivered below. + }, + }; + handlers.end(Object.assign(startEvent, { result: stream })); + await streamPatcherMock.options?.onChunk?.({ + type: "assistant", + message: { + content: [{ text: "Response", type: "text" }], + id: "msg_null_cache", + model: "claude-3-5-sonnet-20241022", + role: "assistant", + // Bedrock, Vertex, and gateway-backed runs report `null` for the + // cache fields they do not populate. + usage: { + cache_creation: null, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + input_tokens: 10, + output_tokens: 40, + }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ type: "result" }); + await streamPatcherMock.options?.onComplete(); + + const llmSpanCallIndex = vi + .mocked(startSpan) + .mock.calls.findIndex( + ([args]) => + args && + typeof args === "object" && + "name" in args && + args.name === "anthropic.messages.create", + ); + expect(llmSpanCallIndex).toBeGreaterThan(-1); + const llmSpan = + vi.mocked(startSpan).mock.results[llmSpanCallIndex]?.value; + expect(llmSpan?.log).toHaveBeenCalledWith( + expect.objectContaining({ + metadata: { + model: "claude-3-5-sonnet-20241022", + provider: "anthropic", + }, + output: [ + { + content: [{ text: "Response", type: "text" }], + role: "assistant", + }, + ], + }), + ); + expect( + vi + .mocked(llmSpan!.log) + .mock.calls.some((call: any[]) => call[0].metrics !== undefined), + ).toBe(false); + }); + + it("layers partial stream usage over the assistant message usage", async () => { + const startEvent = { + arguments: [ + { + prompt: "Test", + options: { + includePartialMessages: true, + model: "claude-3-5-sonnet-20241022", + }, + }, + ], + }; + handlers.start(startEvent); + + const stream = { + async *[Symbol.asyncIterator]() { + // The patcher is mocked; messages are delivered below. + }, + }; + handlers.end(Object.assign(startEvent, { result: stream })); + await streamPatcherMock.options?.onChunk?.({ + type: "stream_event", + event: { + type: "message_start", + // No usable counts here, so the prompt-side totals must come from + // the assistant message rather than defaulting to zero. + message: { id: "msg_partial", usage: { input_tokens: null } }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ + type: "stream_event", + event: { type: "message_delta", usage: { output_tokens: 40 } }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ + type: "assistant", + message: { + content: [{ text: "Response", type: "text" }], + id: "msg_partial", + role: "assistant", + usage: { + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 40, + }, + }, + parent_tool_use_id: null, + }); + await streamPatcherMock.options?.onChunk?.({ + type: "result", + usage: { + cache_creation_input_tokens: 3, + cache_read_input_tokens: 20, + input_tokens: 10, + output_tokens: 40, + }, + }); + await streamPatcherMock.options?.onComplete(); + + const llmSpanCallIndex = vi + .mocked(startSpan) + .mock.calls.findIndex( + ([args]) => + args && + typeof args === "object" && + "name" in args && + args.name === "anthropic.messages.create", + ); + const llmSpan = + vi.mocked(startSpan).mock.results[llmSpanCallIndex]?.value; + expect(llmSpan?.log).toHaveBeenCalledWith( + expect.objectContaining({ + metrics: { + completion_tokens: 40, + prompt_cache_creation_tokens: 3, + prompt_cached_tokens: 20, + prompt_tokens: 33, + tokens: 73, + }, + }), + ); + const taskSpan = vi.mocked(startSpan).mock.results[0]?.value; + expect( + vi + .mocked(taskSpan!.log) + .mock.calls.some((call: any[]) => call[0].metrics !== undefined), + ).toBe(false); + }); }); describe("error handler", () => { diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts index 86237a6df..96f52c9b6 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts +++ b/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts @@ -13,6 +13,8 @@ import { getCurrentUnixTimestamp } from "../../util"; import { extractAnthropicCacheTokens, finalizeAnthropicTokens, + toNumericMetrics, + type AnthropicTokenMetrics, } from "../../wrappers/anthropic-tokens-util"; import { claudeAgentSDKChannels } from "./claude-agent-sdk-channels"; import { CLAUDE_AGENT_SDK_SKIP_LOCAL_TOOL_HOOKS_OPTION } from "./claude-agent-sdk-instrumentation-constants"; @@ -33,6 +35,7 @@ import type { ClaudeAgentSDKMessage, ClaudeAgentSDKQueryOptions, ClaudeAgentSDKQueryParams, + ClaudeAgentSDKUsage, } from "../../vendor-sdk-types/claude-agent-sdk"; type ClaudeConversationMessage = { content: unknown; role: string }; @@ -211,20 +214,89 @@ function seedTaskToolUseIdMapping( } } -function extractUsageFromMessage( - message: ClaudeAgentSDKMessage, -): Record { - const metrics: Record = {}; +function tokenCount(value: unknown): number | undefined { + return typeof value === "number" && + Number.isFinite(value) && + Number.isInteger(value) && + value >= 0 + ? value + : undefined; +} - let usage: unknown; - if (message.type === "assistant") { - usage = message.message?.usage; - } else if (message.type === "result") { - usage = message.usage; +/** + * Snapshots the token fields we understand out of a provider usage object. + * + * Fields degrade individually: Bedrock, Vertex, and gateway-backed runs report + * `null` for cache fields they do not populate, and one unusable field must + * only cost that metric rather than the whole usage object. + */ +function copyUsage(usage: unknown): ClaudeAgentSDKUsage | undefined { + if (!usage || typeof usage !== "object") { + return undefined; } - if (!usage || typeof usage !== "object") { - return metrics; + const copy: ClaudeAgentSDKUsage = {}; + for (const key of [ + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ] as const) { + const value = tokenCount(Reflect.get(usage, key)); + if (value !== undefined) { + copy[key] = value; + } + } + + const cacheCreation = Reflect.get(usage, "cache_creation"); + if (cacheCreation && typeof cacheCreation === "object") { + const cacheCreationCopy: NonNullable< + ClaudeAgentSDKUsage["cache_creation"] + > = {}; + for (const key of [ + "ephemeral_5m_input_tokens", + "ephemeral_1h_input_tokens", + ] as const) { + const value = tokenCount(Reflect.get(cacheCreation, key)); + if (value !== undefined) { + cacheCreationCopy[key] = value; + } + } + if (Object.keys(cacheCreationCopy).length > 0) { + copy.cache_creation = cacheCreationCopy; + } + } + + return Object.keys(copy).length > 0 ? copy : undefined; +} + +/** Layers a newer usage snapshot over an older one, field by field. */ +function mergeUsage( + base: ClaudeAgentSDKUsage | undefined, + override: ClaudeAgentSDKUsage | undefined, +): ClaudeAgentSDKUsage | undefined { + if (!base || !override) { + return override ?? base; + } + + const cacheCreation = + base.cache_creation || override.cache_creation + ? { ...base.cache_creation, ...override.cache_creation } + : undefined; + return { + ...base, + ...override, + ...(cacheCreation && { cache_creation: cacheCreation }), + }; +} + +function extractUsage( + usage: ClaudeAgentSDKUsage | undefined, + includeOutput: boolean, +): Record { + const metrics: AnthropicTokenMetrics = {}; + if (!usage) { + return {}; } const inputTokens = getNumberProperty(usage, "input_tokens"); @@ -232,28 +304,47 @@ function extractUsageFromMessage( metrics.prompt_tokens = inputTokens; } - const outputTokens = getNumberProperty(usage, "output_tokens"); - if (outputTokens !== undefined) { - metrics.completion_tokens = outputTokens; + if (includeOutput) { + const outputTokens = getNumberProperty(usage, "output_tokens"); + if (outputTokens !== undefined) { + metrics.completion_tokens = outputTokens; + } } const cacheReadTokens = getNumberProperty(usage, "cache_read_input_tokens") || 0; const cacheCreationTokens = getNumberProperty(usage, "cache_creation_input_tokens") || 0; + Object.assign( + metrics, + extractAnthropicCacheTokens(cacheReadTokens, cacheCreationTokens), + ); - if (cacheReadTokens > 0 || cacheCreationTokens > 0) { - Object.assign( - metrics, - extractAnthropicCacheTokens(cacheReadTokens, cacheCreationTokens), - ); + const cacheCreation5mTokens = getNumberProperty( + usage.cache_creation, + "ephemeral_5m_input_tokens", + ); + const cacheCreation1hTokens = getNumberProperty( + usage.cache_creation, + "ephemeral_1h_input_tokens", + ); + if (cacheCreation5mTokens !== undefined) { + metrics.prompt_cache_creation_5m_tokens = cacheCreation5mTokens; + } + if (cacheCreation1hTokens !== undefined) { + metrics.prompt_cache_creation_1h_tokens = cacheCreation1hTokens; } - if (Object.keys(metrics).length > 0) { - Object.assign(metrics, finalizeAnthropicTokens(metrics)); + if (Object.keys(metrics).length === 0) { + return {}; } - return metrics; + const finalized = finalizeAnthropicTokens(metrics); + if (metrics.completion_tokens === undefined) { + // A total is only meaningful once both halves are known. + delete finalized.tokens; + } + return toNumericMetrics(finalized); } function buildLLMInput( @@ -325,6 +416,8 @@ async function createLLMSpanForMessages( options: ClaudeAgentSDKQueryOptions, startTime: number, parentSpan: string, + usage: ClaudeAgentSDKUsage | undefined, + hasFinalOutputUsage: boolean, existingSpan?: Span, ): Promise { if (messages.length === 0) { @@ -332,12 +425,17 @@ async function createLLMSpanForMessages( } const lastMessage = messages[messages.length - 1]; - if (lastMessage.type !== "assistant" || !lastMessage.message?.usage) { + // Every assistant message is one provider request and must produce one `llm` + // span. Per-call metrics are attached only when partial messages are enabled; + // otherwise the terminal aggregate belongs exclusively to the task span. + if (lastMessage.type !== "assistant") { return undefined; } - const model = lastMessage.message.model || options.model; - const usage = extractUsageFromMessage(lastMessage); + const model = lastMessage.message?.model || options.model; + const metrics = options.includePartialMessages + ? extractUsage(usage, hasFinalOutputUsage) + : {}; const input = buildLLMInput(promptMessages, conversationHistory); const outputs = messages .map((m) => @@ -368,8 +466,8 @@ async function createLLMSpanForMessages( span.log({ input, - metadata: model ? { model } : undefined, - metrics: usage, + metadata: { ...(model && { model }), provider: "anthropic" }, + ...(Object.keys(metrics).length > 0 ? { metrics } : {}), output: outputs, }); @@ -768,9 +866,6 @@ function createToolTracingHooks( const metadata = { ...subAgentDetailsToMetadata(details), - ...(input.agent_transcript_path && { - "claude_agent_sdk.agent_transcript_path": input.agent_transcript_path, - }), "claude_agent_sdk.stop_hook_active": input.stop_hook_active, }; @@ -864,8 +959,8 @@ function injectTracingHooks( } type QueryState = { - accumulatedOutputTokens: number; activeLlmSpansByParentToolUse: Map; + activePartialMessageIdByParentKey: Map; activeToolSpans: Map; conversationHistoryByParentKey: Map; capturedPromptMessages: ClaudeAgentSDKMessage[] | undefined; @@ -873,6 +968,7 @@ type QueryState = { currentMessageStartTime: number; currentMessages: ClaudeAgentSDKMessage[]; endedSubAgentSpans: Set; + finalOutputUsageMessageIds: Set; finalResults: ClaudeConversationMessage[]; options: ClaudeAgentSDKQueryOptions; originalPrompt: string | AsyncIterable | undefined; @@ -888,6 +984,7 @@ type QueryState = { latestLlmParentBySubAgentToolUse: Map; latestRootLlmParentRef: { value: string | undefined }; toolUseToParent: Map; + usageByMessageId: Map; localToolContext: ClaudeAgentSDKLocalToolContext; }; @@ -947,6 +1044,19 @@ async function finalizeCurrentMessageGroup(state: QueryState): Promise { } } const existingLlmSpan = state.activeLlmSpansByParentToolUse.get(parentKey); + const lastMessage = state.currentMessages[state.currentMessages.length - 1]; + const messageId = lastMessage?.message?.id; + // When partial messages are enabled, stream events carry the freshest + // per-call counts. They can be partial, so layer them over the assistant + // message's usage rather than replacing it. + const usage = state.options.includePartialMessages + ? mergeUsage( + copyUsage(lastMessage?.message?.usage), + messageId ? state.usageByMessageId.get(messageId) : undefined, + ) + : undefined; + const hasFinalOutputUsage = + messageId !== undefined && state.finalOutputUsageMessageIds.has(messageId); const llmSpanResult = await createLLMSpanForMessages( state.currentMessages, @@ -955,6 +1065,8 @@ async function finalizeCurrentMessageGroup(state: QueryState): Promise { state.options, state.currentMessageStartTime, parentSpan, + usage, + hasFinalOutputUsage, existingLlmSpan, ); @@ -979,10 +1091,17 @@ async function finalizeCurrentMessageGroup(state: QueryState): Promise { // fall back to the broader sub-agent task span instead of the LLM span. state.activeLlmSpansByParentToolUse.delete(parentKey); - const lastMessage = state.currentMessages[state.currentMessages.length - 1]; - if (lastMessage?.message?.usage) { - state.accumulatedOutputTokens += - getNumberProperty(lastMessage.message.usage, "output_tokens") || 0; + if (messageId) { + state.usageByMessageId.delete(messageId); + state.finalOutputUsageMessageIds.delete(messageId); + for (const [ + parent, + activeMessageId, + ] of state.activePartialMessageIdByParentKey) { + if (activeMessageId === messageId) { + state.activePartialMessageIdByParentKey.delete(parent); + } + } } state.currentMessages.length = 0; @@ -1128,6 +1247,13 @@ async function ensureActiveLlmSpanForParentToolUse( llmParentSpan = await subAgentSpan.export(); } + // Parent resolution above can yield while a stream message and a tool hook + // race to materialize the same placeholder span. Recheck before creating it. + const racedLlmSpan = activeLlmSpansByParentToolUse.get(parentKey); + if (racedLlmSpan) { + return racedLlmSpan; + } + const llmSpan = startBaseSpan( withSpanInstrumentationName( { @@ -1271,10 +1397,62 @@ async function maybeHandleTaskLifecycleMessage( return true; } +function handlePartialUsageMessage( + state: QueryState, + message: ClaudeAgentSDKMessage, +): boolean { + if (message.type !== "stream_event") { + return false; + } + + const event = message.event; + if (!event || typeof event !== "object") { + return true; + } + + const parentKey = llmParentKey(message.parent_tool_use_id ?? null); + if (event.type === "message_start") { + const messageId = event.message?.id; + const usage = copyUsage(event.message?.usage); + if (messageId) { + state.activePartialMessageIdByParentKey.set(parentKey, messageId); + if (usage) { + state.usageByMessageId.set(messageId, usage); + } + } + return true; + } + + const messageId = state.activePartialMessageIdByParentKey.get(parentKey); + if (!messageId) { + return true; + } + + if (event.type === "message_delta") { + const update = copyUsage(event.usage); + if (update) { + const usage = state.usageByMessageId.get(messageId) ?? {}; + Object.assign(usage, update); + state.usageByMessageId.set(messageId, usage); + if (update.output_tokens !== undefined) { + state.finalOutputUsageMessageIds.add(messageId); + } + } + } else if (event.type === "message_stop") { + state.activePartialMessageIdByParentKey.delete(parentKey); + } + + return true; +} + async function handleStreamMessage( state: QueryState, message: ClaudeAgentSDKMessage, ): Promise { + if (handlePartialUsageMessage(state, message)) { + return; + } + maybeTrackToolUseContext(state, message); if (await maybeHandleTaskLifecycleMessage(state, message)) { return; @@ -1333,45 +1511,10 @@ async function handleStreamMessage( state.currentMessages.push(message); } - if (message.type !== "result" || !message.usage) { + if (message.type !== "result") { return; } - const finalUsageMetrics = extractUsageFromMessage(message); - if ( - state.currentMessages.length > 0 && - finalUsageMetrics.completion_tokens !== undefined - ) { - const lastMessage = state.currentMessages[state.currentMessages.length - 1]; - if (lastMessage?.message?.usage) { - const adjustedTokens = - finalUsageMetrics.completion_tokens - state.accumulatedOutputTokens; - if (adjustedTokens >= 0) { - lastMessage.message.usage.output_tokens = adjustedTokens; - } - - const resultUsage = message.usage; - if (resultUsage && typeof resultUsage === "object") { - const cacheReadTokens = getNumberProperty( - resultUsage, - "cache_read_input_tokens", - ); - if (cacheReadTokens !== undefined) { - lastMessage.message.usage.cache_read_input_tokens = cacheReadTokens; - } - - const cacheCreationTokens = getNumberProperty( - resultUsage, - "cache_creation_input_tokens", - ); - if (cacheCreationTokens !== undefined) { - lastMessage.message.usage.cache_creation_input_tokens = - cacheCreationTokens; - } - } - } - } - const metadata: Record = {}; if (message.num_turns !== undefined) { metadata.num_turns = message.num_turns; @@ -1379,8 +1522,14 @@ async function handleStreamMessage( if (message.session_id !== undefined) { metadata.session_id = message.session_id; } - if (Object.keys(metadata).length > 0) { - state.span.log({ metadata }); + const metrics = state.options.includePartialMessages + ? {} + : extractUsage(copyUsage(message.usage), true); + if (Object.keys(metadata).length > 0 || Object.keys(metrics).length > 0) { + state.span.log({ + ...(Object.keys(metadata).length > 0 ? { metadata } : {}), + ...(Object.keys(metrics).length > 0 ? { metrics } : {}), + }); } } @@ -1410,6 +1559,9 @@ async function finalizeQuerySpan(state: QueryState): Promise { llmSpan.end(); } state.activeLlmSpansByParentToolUse.clear(); + state.activePartialMessageIdByParentKey.clear(); + state.finalOutputUsageMessageIds.clear(); + state.usageByMessageId.clear(); for (const toolSpan of state.activeToolSpans.values()) { toolSpan.end(); @@ -1594,8 +1746,8 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { event.arguments[0] = params; spans.set(event, { - accumulatedOutputTokens: 0, activeLlmSpansByParentToolUse, + activePartialMessageIdByParentKey: new Map(), activeToolSpans, conversationHistoryByParentKey, capturedPromptMessages, @@ -1603,6 +1755,7 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { currentMessageStartTime: startTime, currentMessages: [], endedSubAgentSpans, + finalOutputUsageMessageIds: new Set(), finalResults: [], options: optionsWithHooks, originalPrompt, @@ -1618,6 +1771,7 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { latestLlmParentBySubAgentToolUse, latestRootLlmParentRef, toolUseToParent, + usageByMessageId: new Map(), localToolContext, }); }, diff --git a/js/src/instrumentation/plugins/github-copilot-plugin.ts b/js/src/instrumentation/plugins/github-copilot-plugin.ts index 5a1963a2e..8fa156cfc 100644 --- a/js/src/instrumentation/plugins/github-copilot-plugin.ts +++ b/js/src/instrumentation/plugins/github-copilot-plugin.ts @@ -39,7 +39,7 @@ export function extractMetricsFromUsage(usage: GitHubCopilotUsageData): { metrics: AnthropicTokenMetrics; metadata: Record; } { - const metrics: AnthropicTokenMetrics = { + const rawMetrics: AnthropicTokenMetrics = { prompt_tokens: usage.inputTokens, completion_tokens: usage.outputTokens, ...extractAnthropicCacheTokens( @@ -49,11 +49,13 @@ export function extractMetricsFromUsage(usage: GitHubCopilotUsageData): { }; if (usage.reasoningTokens !== undefined) { - metrics.completion_reasoning_tokens = usage.reasoningTokens; - metrics.reasoning_tokens = usage.reasoningTokens; + rawMetrics.completion_reasoning_tokens = usage.reasoningTokens; + rawMetrics.reasoning_tokens = usage.reasoningTokens; } - Object.assign(metrics, finalizeAnthropicTokens(metrics)); + // Use the returned object: finalization can drop cache-creation metrics, and + // merging it back over `rawMetrics` would keep them. + const metrics = finalizeAnthropicTokens(rawMetrics); const metadata: Record = { model: usage.model, diff --git a/js/src/vendor-sdk-types/anthropic.ts b/js/src/vendor-sdk-types/anthropic.ts index 278e7f392..89cdc644f 100644 --- a/js/src/vendor-sdk-types/anthropic.ts +++ b/js/src/vendor-sdk-types/anthropic.ts @@ -194,10 +194,21 @@ export interface AnthropicUsage { output_tokens: number; cache_read_input_tokens?: number; cache_creation_input_tokens?: number; + cache_creation?: AnthropicCacheCreationUsage | null; server_tool_use?: AnthropicServerToolUseUsage; [key: string]: unknown; } +/** + * Per-TTL breakdown of cache-write tokens. Only recent Anthropic SDK versions + * report this; older ones expose just `cache_creation_input_tokens`. + */ +export interface AnthropicCacheCreationUsage { + ephemeral_5m_input_tokens?: number; + ephemeral_1h_input_tokens?: number; + [key: string]: unknown; +} + export interface AnthropicServerToolUseUsage { web_search_requests?: number; [key: string]: unknown; diff --git a/js/src/vendor-sdk-types/claude-agent-sdk.ts b/js/src/vendor-sdk-types/claude-agent-sdk.ts index c576993bf..6f33d84bd 100644 --- a/js/src/vendor-sdk-types/claude-agent-sdk.ts +++ b/js/src/vendor-sdk-types/claude-agent-sdk.ts @@ -5,11 +5,24 @@ */ // Shared usage shape used in message and result -interface Usage { +export interface ClaudeAgentSDKUsage { input_tokens?: number; output_tokens?: number; cache_read_input_tokens?: number; cache_creation_input_tokens?: number; + cache_creation?: { + ephemeral_5m_input_tokens?: number; + ephemeral_1h_input_tokens?: number; + }; +} + +interface ClaudeAgentSDKRawStreamEvent { + type?: string; + message?: { + id?: string; + usage?: ClaudeAgentSDKUsage; + }; + usage?: ClaudeAgentSDKUsage; } interface TaskUsage { @@ -37,10 +50,11 @@ export interface ClaudeAgentSDKMessage { role?: string; content?: unknown; model?: string; - usage?: Usage; + usage?: ClaudeAgentSDKUsage; }; + event?: ClaudeAgentSDKRawStreamEvent; parent_tool_use_id?: string | null; - usage?: Usage | TaskUsage; + usage?: ClaudeAgentSDKUsage | TaskUsage; num_turns?: number; session_id?: string; task_id?: string; @@ -75,6 +89,7 @@ export interface ClaudeAgentSDKQueryOptions { debug?: boolean; agentName?: string; instructions?: string; + includePartialMessages?: boolean; mcpServers?: ClaudeAgentSDKMcpServersConfig; hooks?: Record; [key: string]: unknown; diff --git a/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts b/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts index 5c12e8b39..d483ed02b 100644 --- a/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts +++ b/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts @@ -9,6 +9,7 @@ import { import { extractAnthropicCacheTokens, finalizeAnthropicTokens, + toNumericMetrics, } from "../../anthropic-tokens-util"; import { processInputAttachments } from "../../attachment-utils"; @@ -129,7 +130,9 @@ function normalizeUsageMetrics( ); Object.assign(metrics, cacheTokens); - Object.assign(metrics, finalizeAnthropicTokens(metrics)); + // Use the returned object: finalization can drop cache-creation metrics, + // and merging it back over `metrics` would keep them. + return toNumericMetrics(finalizeAnthropicTokens(metrics)); } } diff --git a/js/src/wrappers/anthropic-tokens-util.test.ts b/js/src/wrappers/anthropic-tokens-util.test.ts new file mode 100644 index 000000000..b7f1fcc55 --- /dev/null +++ b/js/src/wrappers/anthropic-tokens-util.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { + finalizeAnthropicTokens, + toNumericMetrics, +} from "./anthropic-tokens-util"; + +describe("finalizeAnthropicTokens", () => { + it("includes legacy aggregate cache creation tokens", () => { + expect( + finalizeAnthropicTokens({ + completion_tokens: 48, + prompt_cache_creation_tokens: 199, + prompt_cached_tokens: 24_243, + prompt_tokens: 8, + }), + ).toMatchObject({ + prompt_tokens: 24_450, + tokens: 24_498, + }); + }); + + it("uses TTL cache creation tokens instead of the legacy aggregate", () => { + const metrics = finalizeAnthropicTokens({ + completion_tokens: 48, + prompt_cache_creation_1h_tokens: 0, + prompt_cache_creation_5m_tokens: 199, + prompt_cache_creation_tokens: 199, + prompt_cached_tokens: 24_243, + prompt_tokens: 8, + }); + + expect(metrics).toMatchObject({ + prompt_tokens: 24_450, + tokens: 24_498, + }); + expect(metrics.prompt_cache_creation_tokens).toBeUndefined(); + }); + + it("keeps the aggregate when the TTL cache creation breakdown is partial", () => { + expect( + finalizeAnthropicTokens({ + completion_tokens: 48, + prompt_cache_creation_5m_tokens: 199, + prompt_cache_creation_tokens: 999, + prompt_cached_tokens: 24_243, + prompt_tokens: 8, + }), + ).toMatchObject({ + prompt_cache_creation_5m_tokens: 199, + prompt_cache_creation_tokens: 999, + prompt_tokens: 25_250, + tokens: 25_298, + }); + }); + + it("drops a fully represented aggregate for callers that keep the returned object", () => { + const raw = { + completion_tokens: 48, + prompt_cache_creation_5m_tokens: 199, + prompt_cache_creation_tokens: 199, + prompt_tokens: 8, + }; + + // Merging the result back over the input would keep both representations, + // which Anthropic spans must never carry. + expect( + Object.assign({ ...raw }, finalizeAnthropicTokens(raw)), + ).toHaveProperty("prompt_cache_creation_tokens"); + expect(finalizeAnthropicTokens(raw)).not.toHaveProperty( + "prompt_cache_creation_tokens", + ); + }); +}); + +describe("toNumericMetrics", () => { + it("drops unset metrics", () => { + expect( + toNumericMetrics({ + completion_tokens: 0, + prompt_cache_creation_tokens: undefined, + prompt_tokens: 8, + }), + ).toEqual({ completion_tokens: 0, prompt_tokens: 8 }); + }); +}); diff --git a/js/src/wrappers/anthropic-tokens-util.ts b/js/src/wrappers/anthropic-tokens-util.ts index f5cd84aac..ee5efffd3 100644 --- a/js/src/wrappers/anthropic-tokens-util.ts +++ b/js/src/wrappers/anthropic-tokens-util.ts @@ -9,23 +9,65 @@ export interface AnthropicTokenMetrics { completion_tokens?: number; prompt_cached_tokens?: number; prompt_cache_creation_tokens?: number; + prompt_cache_creation_5m_tokens?: number; + prompt_cache_creation_1h_tokens?: number; tokens?: number; [key: string]: number | undefined; } +/** + * Rolls cache tokens back into `prompt_tokens`/`tokens` and removes the + * aggregate cache-creation metric only when the per-TTL breakdown accounts for + * all reported cache-write tokens. + * + * Callers MUST use the returned object rather than `Object.assign`-ing it back + * over the input because finalization may drop + * `prompt_cache_creation_tokens`. + */ export function finalizeAnthropicTokens( metrics: AnthropicTokenMetrics, ): AnthropicTokenMetrics { + const hasSplitCacheCreationTokens = + metrics.prompt_cache_creation_5m_tokens !== undefined || + metrics.prompt_cache_creation_1h_tokens !== undefined; + const splitCacheCreationTokens = + (metrics.prompt_cache_creation_5m_tokens || 0) + + (metrics.prompt_cache_creation_1h_tokens || 0); + const aggregateCacheCreationTokens = + metrics.prompt_cache_creation_tokens || 0; + const effectiveCacheCreationTokens = Math.max( + aggregateCacheCreationTokens, + splitCacheCreationTokens, + ); const prompt_tokens = (metrics.prompt_tokens || 0) + (metrics.prompt_cached_tokens || 0) + - (metrics.prompt_cache_creation_tokens || 0); + effectiveCacheCreationTokens; - return { + const finalized = { ...metrics, prompt_tokens, tokens: prompt_tokens + (metrics.completion_tokens || 0), }; + // Keep the aggregate as a pricing fallback when the split is partial. + if ( + hasSplitCacheCreationTokens && + splitCacheCreationTokens >= aggregateCacheCreationTokens + ) { + delete finalized.prompt_cache_creation_tokens; + } + return finalized; +} + +/** Drops unset metrics so the result can be logged as span metrics. */ +export function toNumericMetrics( + metrics: AnthropicTokenMetrics, +): Record { + return Object.fromEntries( + Object.entries(metrics).filter( + (entry): entry is [string, number] => entry[1] !== undefined, + ), + ); } export function extractAnthropicCacheTokens(