From 7d41d4e1f23d125a82fb87c6fa3cefae471c0960 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Wed, 12 Aug 2026 23:05:06 -0600 Subject: [PATCH 1/4] VCR --- .../__files/v1_messages-74a60d2aa4be.txt | 36 +++++++++++++ .../mappings/v1_messages-74a60d2aa4be.json | 53 +++++++++++++++++++ .../__files/chat_completions-1396042660a1.txt | 22 ++++++++ .../chat_completions-1396042660a1.json | 49 +++++++++++++++++ 4 files changed, 160 insertions(+) create mode 100644 test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-74a60d2aa4be.txt create mode 100644 test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-74a60d2aa4be.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-1396042660a1.txt create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-1396042660a1.json diff --git a/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-74a60d2aa4be.txt b/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-74a60d2aa4be.txt new file mode 100644 index 00000000..c84c346a --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-74a60d2aa4be.txt @@ -0,0 +1,36 @@ +event: message_start +data: {"type":"message_start","message":{"model":"claude-haiku-4-5-20251001","id":"msg_011Ce1Gh1KcCEtDutfBsT34A","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":596,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":22,"service_tier":"standard","inference_geo":"not_available"}} } + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_011jMmPnwD3TqUjMg6Kpdo3P","name":"get_weather","input":{},"caller":{"type":"direct"}} } + +event: ping +data: {"type": "ping"} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":""} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"loca"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"tion\": \"Pa"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"ris,"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":" France"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"}"} } + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null,"stop_details":null},"usage":{"input_tokens":596,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":56} } + +event: message_stop +data: {"type":"message_stop" } + diff --git a/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-74a60d2aa4be.json b/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-74a60d2aa4be.json new file mode 100644 index 00000000..308e76f4 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-74a60d2aa4be.json @@ -0,0 +1,53 @@ +{ + "id" : "2c7344ba-cce7-35f0-9067-9cf5f984f5f2", + "name" : "v1_messages", + "request" : { + "url" : "/v1/messages", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"model\":\"claude-haiku-4-5\",\"messages\":[{\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris, France?\"}],\"role\":\"user\"}],\"system\":\"\",\"max_tokens\":1024,\"stream\":true,\"temperature\":0.0,\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get the current weather for a location\",\"input_schema\":{\"$schema\":\"https://json-schema.org/draft/2020-12/schema\",\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"],\"additionalProperties\":false}}]}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_messages-74a60d2aa4be.txt", + "headers" : { + "anthropic-organization-id" : "27796668-7351-40ac-acc4-024aee8995a5", + "Server" : "cloudflare", + "vary" : "Accept-Encoding", + "anthropic-ratelimit-output-tokens-limit" : "2000000", + "anthropic-ratelimit-output-tokens-reset" : "2026-08-13T19:06:42Z", + "anthropic-ratelimit-input-tokens-reset" : "2026-08-13T19:06:42Z", + "anthropic-ratelimit-tokens-remaining" : "12000000", + "anthropic-ratelimit-requests-limit" : "20000", + "anthropic-workspace-id" : "wrkspc_01J8hau4yJ2SKvtbeP9drm9A", + "Content-Security-Policy" : "default-src 'none'; frame-ancestors 'none'", + "anthropic-ratelimit-input-tokens-remaining" : "10000000", + "anthropic-ratelimit-requests-remaining" : "19999", + "Content-Type" : "text/event-stream; charset=utf-8", + "CF-RAY" : "a2a9ffa02882fcac-SEA", + "anthropic-ratelimit-tokens-limit" : "12000000", + "cf-cache-status" : "DYNAMIC", + "request-id" : "req_011Ce1Ggzn7i9DPduqfs8VAq", + "anthropic-ratelimit-tokens-reset" : "2026-08-13T19:06:42Z", + "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", + "Date" : "Thu, 13 Aug 2026 19:06:43 GMT", + "X-Robots-Tag" : "none", + "anthropic-ratelimit-requests-reset" : "2026-08-13T19:06:42Z", + "Cache-Control" : "no-cache", + "anthropic-ratelimit-input-tokens-limit" : "10000000", + "traceresponse" : "00-fd09b9dfb45db31fcdbcdc495ad7eae1-b20fd450b01eebf8-01", + "anthropic-ratelimit-output-tokens-remaining" : "2000000" + } + }, + "uuid" : "2c7344ba-cce7-35f0-9067-9cf5f984f5f2", + "persistent" : true, + "insertionIndex" : 25 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-1396042660a1.txt b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-1396042660a1.txt new file mode 100644 index 00000000..6c25ac8b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-1396042660a1.txt @@ -0,0 +1,22 @@ +data: {"id":"chatcmpl-ECHdUuZ1LG7sB6Y2DgfxvKaYsOglO","object":"chat.completion.chunk","created":1786596436,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_788bf556eb","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_wuPFCcmtaOVhAP6HoGwBydJm","type":"function","function":{"name":"get_weather","arguments":""}}],"refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Qs2m"} + +data: {"id":"chatcmpl-ECHdUuZ1LG7sB6Y2DgfxvKaYsOglO","object":"chat.completion.chunk","created":1786596436,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_788bf556eb","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"I3Tu123zyIl7h5"} + +data: {"id":"chatcmpl-ECHdUuZ1LG7sB6Y2DgfxvKaYsOglO","object":"chat.completion.chunk","created":1786596436,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_788bf556eb","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"location"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"EyFMyF9je"} + +data: {"id":"chatcmpl-ECHdUuZ1LG7sB6Y2DgfxvKaYsOglO","object":"chat.completion.chunk","created":1786596436,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_788bf556eb","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"3tId1bVwBMpn"} + +data: {"id":"chatcmpl-ECHdUuZ1LG7sB6Y2DgfxvKaYsOglO","object":"chat.completion.chunk","created":1786596436,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_788bf556eb","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"Paris"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Phhvhbp3j2qD"} + +data: {"id":"chatcmpl-ECHdUuZ1LG7sB6Y2DgfxvKaYsOglO","object":"chat.completion.chunk","created":1786596436,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_788bf556eb","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":","}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":""} + +data: {"id":"chatcmpl-ECHdUuZ1LG7sB6Y2DgfxvKaYsOglO","object":"chat.completion.chunk","created":1786596436,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_788bf556eb","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" France"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"DTf6nWvuPs"} + +data: {"id":"chatcmpl-ECHdUuZ1LG7sB6Y2DgfxvKaYsOglO","object":"chat.completion.chunk","created":1786596436,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_788bf556eb","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"kyjcWSXPAaohmP"} + +data: {"id":"chatcmpl-ECHdUuZ1LG7sB6Y2DgfxvKaYsOglO","object":"chat.completion.chunk","created":1786596436,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_788bf556eb","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":null,"obfuscation":"GgWcuTsiXrRhr24"} + +data: {"id":"chatcmpl-ECHdUuZ1LG7sB6Y2DgfxvKaYsOglO","object":"chat.completion.chunk","created":1786596436,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_788bf556eb","choices":[],"usage":{"prompt_tokens":54,"completion_tokens":16,"total_tokens":70,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"YxWeDMJHuo"} + +data: [DONE] + diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-1396042660a1.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-1396042660a1.json new file mode 100644 index 00000000..89392a1e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-1396042660a1.json @@ -0,0 +1,49 @@ +{ + "id" : "53345a74-6b35-38d0-a840-77a7ee136e13", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"What is the weather in Paris, France?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":256,\"stream\":true,\"stream_options\":{\"include_usage\":true},\"temperature\":0.0,\"tools\":[{\"type\":\"function\",\"function\":{\"description\":\"Get the current weather for a location\",\"name\":\"get_weather\",\"parameters\":{\"$schema\":\"https://json-schema.org/draft/2020-12/schema\",\"additionalProperties\":false,\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-1396042660a1.txt", + "headers" : { + "x-request-id" : "req_db5e36cdfe8843f88bb7cdfefac85571", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2a514a898693295-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999987", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Thu, 13 Aug 2026 04:47:16 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=T6xnzzsEEIsgU4dO39AMuSME3vsE4DxJdP2FpWMPz88-1786596435.2965999-1.0.1.1-ruWJF7ZKD1_jsgt72ebauZR9RTzTOSM6XPzwRP6AOFwzBS2DCLrpVdb3nh1ehixaAAUiRcmfQiFq8mU8QCbvjF5X5V4YtRpV_yKlAktbpyFvctNs51wzXG77GCi.oQTu; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 13 Aug 2026 05:17:16 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "482", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "text/event-stream; charset=utf-8" + } + }, + "uuid" : "53345a74-6b35-38d0-a840-77a7ee136e13", + "persistent" : true, + "insertionIndex" : 35 +} \ No newline at end of file From f40fe2d119add2d527c997f11f171a2022bba8c3 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Wed, 12 Aug 2026 21:35:10 -0600 Subject: [PATCH 2/4] fix muzzle range latest version of google genai doesn't bundle slf4j-api. Will address a deeper fix in a follow-up --- braintrust-sdk/instrumentation/genai_1_18_0/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/braintrust-sdk/instrumentation/genai_1_18_0/build.gradle b/braintrust-sdk/instrumentation/genai_1_18_0/build.gradle index 4c4e65fd..bff7ab42 100644 --- a/braintrust-sdk/instrumentation/genai_1_18_0/build.gradle +++ b/braintrust-sdk/instrumentation/genai_1_18_0/build.gradle @@ -5,7 +5,8 @@ muzzle { pass { group = 'com.google.genai' module = 'google-genai' - versions = '[1.18.0,)' + // TODO: autoinstrumentation muzzle fails in 1.65.0 because slf4j api is no longer a transitive dep + versions = '[1.18.0,1.65.0)' } } From c98c40cf36b625a7413c6537901bf461bed7f2d7 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Wed, 12 Aug 2026 21:36:25 -0600 Subject: [PATCH 3/4] capture reasoning tokens in openai completions api --- .../braintrust/instrumentation/InstrumentationSemConv.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java index b0a0b036..1dc25cfe 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java @@ -171,6 +171,13 @@ private static void tagOpenAIResponse( metrics.put("prompt_cached_tokens", details.get("cached_tokens")); } } + // Reasoning tokens (Chat Completions API) + if (usage.has("completion_tokens_details")) { + JsonNode details = usage.get("completion_tokens_details"); + if (details.has("reasoning_tokens")) { + metrics.put("completion_reasoning_tokens", details.get("reasoning_tokens")); + } + } // Responses API field names if (usage.has("input_tokens")) metrics.put("prompt_tokens", usage.get("input_tokens")); if (usage.has("output_tokens")) From ff7cf88b948f8c01ab0f8d88c0372c149760f53b Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Wed, 12 Aug 2026 21:37:50 -0600 Subject: [PATCH 4/4] response accumulator util --- .../langchain/v1_8_0/WrappedHttpClient.java | 54 +---- .../auto/LangchainInstrumentationModule.java | 1 + .../v1_8_0/BraintrustLangchainTest.java | 129 +++++++++++ .../openai/v2_15_0/BraintrustOpenAITest.java | 178 +++++++++++++++ .../springai/v1_0_0/BraintrustSpringAI.java | 215 +++++++++--------- ...pringAIAnthropicInstrumentationModule.java | 1 + .../SpringAIOpenAIInstrumentationModule.java | 1 + .../v1_0_0/BraintrustSpringAITest.java | 136 +++++++++++ .../SseResponseAccumulator.java | 170 ++++++++++++++ .../SseResponseAccumulatorTest.java | 133 +++++++++++ .../__files/chat_completions-538b5ad9c2fe.txt | 22 ++ .../__files/chat_completions-acc477a295ad.txt | 22 ++ .../openai/__files/responses-9a580e94f540.txt | 39 ++++ .../chat_completions-538b5ad9c2fe.json | 49 ++++ .../chat_completions-acc477a295ad.json | 49 ++++ .../mappings/responses-9a580e94f540.json | 42 ++++ 16 files changed, 1082 insertions(+), 159 deletions(-) create mode 100644 braintrust-sdk/src/main/java/dev/braintrust/instrumentation/SseResponseAccumulator.java create mode 100644 braintrust-sdk/src/test/java/dev/braintrust/instrumentation/SseResponseAccumulatorTest.java create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-538b5ad9c2fe.txt create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-acc477a295ad.txt create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-9a580e94f540.txt create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-538b5ad9c2fe.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-acc477a295ad.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-9a580e94f540.json diff --git a/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/WrappedHttpClient.java b/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/WrappedHttpClient.java index b0084fe4..cfeeec89 100644 --- a/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/WrappedHttpClient.java +++ b/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/WrappedHttpClient.java @@ -1,10 +1,8 @@ package dev.braintrust.instrumentation.langchain.v1_8_0; -import static dev.braintrust.json.BraintrustJsonMapper.toJson; - -import com.fasterxml.jackson.databind.JsonNode; import dev.braintrust.bootstrap.BraintrustBridge; import dev.braintrust.instrumentation.InstrumentationSemConv; +import dev.braintrust.instrumentation.SseResponseAccumulator; import dev.braintrust.json.BraintrustJsonMapper; import dev.langchain4j.exception.HttpException; import dev.langchain4j.http.client.HttpClient; @@ -126,9 +124,8 @@ static class WrappedServerSentEventListener implements ServerSentEventListener { private final String providerName; private final long startNanos = System.nanoTime(); private final AtomicLong timeToFirstTokenNanos = new AtomicLong(); - private final StringBuilder contentBuffer = new StringBuilder(); - private String finishReason = null; - private JsonNode usageData = null; + private final SseResponseAccumulator accumulator = + new SseResponseAccumulator(BraintrustJsonMapper.get()); WrappedServerSentEventListener( ServerSentEventListener delegate, Span span, String providerName) { @@ -182,52 +179,17 @@ public void onClose() { private void accumulateChunk(String data) { if (data == null || data.isEmpty() || "[DONE]".equals(data)) return; - try { - if (timeToFirstTokenNanos.get() == 0L) { - timeToFirstTokenNanos.compareAndExchange(0L, System.nanoTime() - startNanos); - } - JsonNode chunk = BraintrustJsonMapper.get().readTree(data); - if (chunk.has("choices") && chunk.get("choices").size() > 0) { - JsonNode choice = chunk.get("choices").get(0); - if (choice.has("delta")) { - JsonNode delta = choice.get("delta"); - if (delta.has("content")) { - contentBuffer.append(delta.get("content").asText()); - } - } - if (choice.has("finish_reason") && !choice.get("finish_reason").isNull()) { - finishReason = choice.get("finish_reason").asText(); - } - } - if (chunk.has("usage") && !chunk.get("usage").isNull()) { - usageData = chunk.get("usage"); - } - } catch (Exception e) { - log.debug("Failed to parse SSE chunk: {}", data, e); + if (timeToFirstTokenNanos.get() == 0L) { + timeToFirstTokenNanos.compareAndExchange(0L, System.nanoTime() - startNanos); } + accumulator.merge(data); } private void finalizeSpan() { try { - var root = BraintrustJsonMapper.get().createObjectNode(); - - var choicesArray = BraintrustJsonMapper.get().createArrayNode(); - var choice = BraintrustJsonMapper.get().createObjectNode(); - choice.put("index", 0); - if (finishReason != null) choice.put("finish_reason", finishReason); - var message = BraintrustJsonMapper.get().createObjectNode(); - message.put("role", "assistant"); - message.put("content", contentBuffer.toString()); - choice.set("message", message); - choicesArray.add(choice); - root.set("choices", choicesArray); - - if (usageData != null) { - root.set("usage", usageData); - } - Long ttft = timeToFirstTokenNanos.get(); - InstrumentationSemConv.tagLLMSpanResponse(span, providerName, toJson(root), ttft); + InstrumentationSemConv.tagLLMSpanResponse( + span, providerName, accumulator.build(), ttft); } catch (Exception e) { log.debug("Failed to finalize streaming span", e); } diff --git a/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/auto/LangchainInstrumentationModule.java b/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/auto/LangchainInstrumentationModule.java index 4e478fd0..63ddc6a0 100644 --- a/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/auto/LangchainInstrumentationModule.java +++ b/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/auto/LangchainInstrumentationModule.java @@ -36,6 +36,7 @@ public List getHelperClassNames() { MANUAL_PACKAGE + "TracingProxy", MANUAL_PACKAGE + "TracingToolExecutor", MANUAL_PACKAGE + "OtelContextPassingExecutor", + "dev.braintrust.instrumentation.SseResponseAccumulator", "dev.braintrust.instrumentation.InstrumentationSemConv", "dev.braintrust.json.BraintrustJsonMapper"); } diff --git a/braintrust-sdk/instrumentation/langchain_1_8_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_8_0/BraintrustLangchainTest.java b/braintrust-sdk/instrumentation/langchain_1_8_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_8_0/BraintrustLangchainTest.java index 93dad1e2..e5756cfc 100644 --- a/braintrust-sdk/instrumentation/langchain_1_8_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_8_0/BraintrustLangchainTest.java +++ b/braintrust-sdk/instrumentation/langchain_1_8_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_8_0/BraintrustLangchainTest.java @@ -7,9 +7,12 @@ import dev.braintrust.TestHarness; import dev.braintrust.instrumentation.Instrumenter; import dev.langchain4j.agent.tool.Tool; +import dev.langchain4j.agent.tool.ToolSpecification; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.ChatModel; import dev.langchain4j.model.chat.StreamingChatModel; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.request.json.JsonObjectSchema; import dev.langchain4j.model.chat.response.ChatResponse; import dev.langchain4j.model.chat.response.StreamingChatResponseHandler; import dev.langchain4j.model.openai.OpenAiChatModel; @@ -109,6 +112,9 @@ void testSyncChatCompletion() { assertNotNull( output.get(0).get("message").get("content"), "Output should contain assistant response content"); + + // The serialized span output should reflect the full response the client received. + assertSpanOutputReflects(response, span); } @Test @@ -239,6 +245,129 @@ public void onError(Throwable error) { choice.get("message").get("content"), "Output should contain the complete streamed response"); assertNotNull(choice.get("finish_reason"), "Output should have finish_reason"); + + // The reconstructed streaming span output should reflect the full response the client + // received — the instrumentation must feed every SSE event to the accumulator. + assertSpanOutputReflects(response, llmSpan); + } + + @Test + @SneakyThrows + void testStreamingChatCompletionWithTools() { + // Auto-instrumentation intercepts OpenAiStreamingChatModel.Builder.build() + StreamingChatModel model = + OpenAiStreamingChatModel.builder() + .apiKey(testHarness.openAiApiKey()) + .baseUrl(testHarness.openAiBaseUrl()) + .modelName("gpt-4o") + .temperature(0.0) + .build(); + + var weatherTool = + ToolSpecification.builder() + .name("get_weather") + .description("Get the current weather for a location") + .parameters( + JsonObjectSchema.builder() + .addStringProperty( + "location", + "The city and state, e.g. San" + " Francisco, CA") + .required("location") + .build()) + .build(); + + var chatRequest = + ChatRequest.builder() + .messages(UserMessage.from("What is the weather in Paris, France?")) + .toolSpecifications(weatherTool) + .build(); + + var future = new CompletableFuture(); + model.chat( + chatRequest, + new StreamingChatResponseHandler() { + @Override + public void onPartialResponse(String token) {} + + @Override + public void onCompleteResponse(ChatResponse response) { + future.complete(response); + } + + @Override + public void onError(Throwable error) { + future.completeExceptionally(error); + } + }); + var response = future.get(); + + // The stream must carry tool-call deltas (merged by index) all the way to the span — the + // original bug dropped tool_calls entirely from streaming reconstruction. + assertTrue( + response.aiMessage().hasToolExecutionRequests(), + "Model should have requested a tool call"); + + var llmSpan = + testHarness.awaitExportedSpans(1).stream() + .filter(s -> s.getName().equals("Chat Completion")) + .findFirst() + .orElseThrow(() -> new AssertionError("no 'Chat Completion' llm span")); + + assertSpanOutputReflects(response, llmSpan); + } + + /** + * Asserts that the llm span's serialized output ({@code braintrust.output_json}) reflects the + * full response the langchain client received — comparing the reconstructed assistant message + * against the client's parsed {@link ChatResponse} (content, thinking, and tool calls) rather + * than hand-asserting individual fields per test. langchain decodes the same stream + * independently of our accumulator, so agreement is a meaningful end-to-end check. + */ + @SneakyThrows + private void assertSpanOutputReflects(ChatResponse clientResponse, SpanData llmSpan) { + String outputJson = + llmSpan.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); + assertNotNull(outputJson, "Span should have braintrust.output_json"); + JsonNode message = JSON_MAPPER.readTree(outputJson).get(0).get("message"); + assertNotNull(message, "Span output should contain a choice message"); + + var aiMessage = clientResponse.aiMessage(); + + if (aiMessage.text() != null) { + assertEquals( + aiMessage.text(), + message.path("content").asText(), + "Span output content should match the client's assistant text"); + } + if (aiMessage.thinking() != null) { + assertEquals( + aiMessage.thinking(), + message.path("reasoning_content").asText(), + "Span output reasoning_content should match the client's thinking"); + } + if (aiMessage.hasToolExecutionRequests()) { + JsonNode toolCalls = message.get("tool_calls"); + assertNotNull(toolCalls, "Span output should contain tool_calls"); + var requests = aiMessage.toolExecutionRequests(); + assertEquals( + requests.size(), toolCalls.size(), "tool_calls count should match the client"); + for (int i = 0; i < requests.size(); i++) { + var request = requests.get(i); + JsonNode function = toolCalls.get(i).get("function"); + assertEquals( + request.name(), function.get("name").asText(), "tool name should match"); + assertEquals( + JSON_MAPPER.readTree(request.arguments()), + JSON_MAPPER.readTree(function.get("arguments").asText()), + "tool arguments should match"); + if (request.id() != null) { + assertEquals( + request.id(), + toolCalls.get(i).get("id").asText(), + "tool id should match"); + } + } + } } @Test diff --git a/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java index b29f8199..b34e5fc7 100644 --- a/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java +++ b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java @@ -8,20 +8,25 @@ import com.openai.client.OpenAIClientAsync; import com.openai.client.okhttp.OpenAIOkHttpClient; import com.openai.client.okhttp.OpenAIOkHttpClientAsync; +import com.openai.core.JsonValue; import com.openai.core.http.StreamResponse; import com.openai.helpers.ChatCompletionAccumulator; import com.openai.helpers.ResponseAccumulator; import com.openai.models.ChatModel; +import com.openai.models.FunctionDefinition; +import com.openai.models.FunctionParameters; import com.openai.models.Reasoning; import com.openai.models.ReasoningEffort; import com.openai.models.chat.completions.ChatCompletionCreateParams; import com.openai.models.chat.completions.ChatCompletionStreamOptions; +import com.openai.models.chat.completions.ChatCompletionTool; import com.openai.models.responses.*; import dev.braintrust.TestHarness; import dev.braintrust.instrumentation.Instrumenter; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.sdk.trace.data.SpanData; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import lombok.SneakyThrows; import net.bytebuddy.agent.ByteBuddyAgent; @@ -439,6 +444,179 @@ void testWrappedClientObjectContract() { "toString should identify the proxy, got: " + wrapped); } + @Test + @SneakyThrows + void testCompletionsStreamingWithTools() { + OpenAIClient openAIClient = + OpenAIOkHttpClient.builder() + .baseUrl(testHarness.openAiBaseUrl()) + .apiKey(testHarness.openAiApiKey()) + .build(); + + var weatherTool = + ChatCompletionTool.builder() + .function( + FunctionDefinition.builder() + .name("get_weather") + .description("Get the current weather for a location") + .parameters(weatherParameters(FunctionParameters.builder())) + .build()) + .build(); + + var request = + ChatCompletionCreateParams.builder() + .model(ChatModel.GPT_4O) + .addUserMessage("What is the weather in Paris, France?") + .temperature(0.0) + .addTool(weatherTool) + .streamOptions( + ChatCompletionStreamOptions.builder().includeUsage(true).build()) + .build(); + + var accumulator = ChatCompletionAccumulator.create(); + try (var stream = openAIClient.chat().completions().createStreaming(request)) { + stream.stream().forEach(accumulator::accumulate); + } + var toolCalls = accumulator.chatCompletion().choices().get(0).message().toolCalls(); + assertTrue(toolCalls.isPresent() && !toolCalls.get().isEmpty(), "model should call a tool"); + + var spans = testHarness.awaitExportedSpans(); + assertEquals(1, spans.size()); + var span = spans.get(0); + assertValidOpenAISpan(span, true); + + // The reconstructed streaming span output must carry the same tool calls the client saw. + JsonNode spanToolCalls = spanOutput(span).get(0).get("message").get("tool_calls"); + assertNotNull(spanToolCalls, "streaming span output should contain tool_calls"); + assertEquals(toolCalls.get().size(), spanToolCalls.size(), "tool_calls count should match"); + for (int i = 0; i < toolCalls.get().size(); i++) { + var tc = toolCalls.get().get(i); + JsonNode spanFn = spanToolCalls.get(i).get("function"); + assertEquals( + tc.function().name(), spanFn.get("name").asText(), "tool name should match"); + assertEquals( + JSON_MAPPER.readTree(tc.function().arguments()), + JSON_MAPPER.readTree(spanFn.get("arguments").asText()), + "tool arguments should match"); + assertEquals(tc.id(), spanToolCalls.get(i).get("id").asText(), "tool id should match"); + } + } + + @Test + @SneakyThrows + void testResponsesStreamingWithTools() { + OpenAIClient client = + OpenAIOkHttpClient.builder() + .baseUrl(testHarness.openAiBaseUrl()) + .apiKey(testHarness.openAiApiKey()) + .build(); + + var weatherTool = + FunctionTool.builder() + .name("get_weather") + .description("Get the current weather for a location") + .parameters(weatherParameters(FunctionTool.Parameters.builder())) + .strict(false) + .build(); + + var request = + ResponseCreateParams.builder() + .model(ChatModel.GPT_4O) + .inputOfResponse( + List.of( + ResponseInputItem.ofEasyInputMessage( + EasyInputMessage.builder() + .role(EasyInputMessage.Role.USER) + .content( + "What is the weather in Paris," + + " France?") + .build()))) + .addTool(weatherTool) + .build(); + + var accumulator = ResponseAccumulator.create(); + try (var stream = client.responses().createStreaming(request)) { + stream.stream().forEach(accumulator::accumulate); + } + var functionCalls = + accumulator.response().output().stream() + .filter(ResponseOutputItem::isFunctionCall) + .map(ResponseOutputItem::asFunctionCall) + .toList(); + assertFalse(functionCalls.isEmpty(), "model should call a function tool"); + + var spans = testHarness.awaitExportedSpans(); + assertEquals(1, spans.size()); + var span = spans.get(0); + assertValidOpenAISpan(span, true); + + // The Responses API serializes output as an "output" array; the streamed function_call + // items must survive reconstruction into the span exactly as the client accumulated them. + var spanFunctionCalls = new java.util.ArrayList(); + spanOutput(span) + .forEach( + item -> { + if ("function_call".equals(item.path("type").asText())) { + spanFunctionCalls.add(item); + } + }); + assertEquals( + functionCalls.size(), spanFunctionCalls.size(), "function_call count should match"); + for (int i = 0; i < functionCalls.size(); i++) { + var fc = functionCalls.get(i); + JsonNode spanFc = spanFunctionCalls.get(i); + assertEquals(fc.name(), spanFc.get("name").asText(), "function name should match"); + assertEquals( + JSON_MAPPER.readTree(fc.arguments()), + JSON_MAPPER.readTree(spanFc.get("arguments").asText()), + "function arguments should match"); + assertEquals(fc.callId(), spanFc.get("call_id").asText(), "call_id should match"); + } + } + + /** A minimal {@code get_weather(location)} JSON-schema, shared by both tool-call tests. */ + private static FunctionParameters weatherParameters(FunctionParameters.Builder builder) { + return builder.putAdditionalProperty("type", JsonValue.from("object")) + .putAdditionalProperty( + "properties", + JsonValue.from( + Map.of( + "location", + Map.of( + "type", + "string", + "description", + "The city and state")))) + .putAdditionalProperty("required", JsonValue.from(List.of("location"))) + .build(); + } + + /** Same {@code get_weather} schema for the Responses API's parameter type. */ + private static FunctionTool.Parameters weatherParameters( + FunctionTool.Parameters.Builder builder) { + return builder.putAdditionalProperty("type", JsonValue.from("object")) + .putAdditionalProperty( + "properties", + JsonValue.from( + Map.of( + "location", + Map.of( + "type", + "string", + "description", + "The city and state")))) + .putAdditionalProperty("required", JsonValue.from(List.of("location"))) + .build(); + } + + @SneakyThrows + private JsonNode spanOutput(SpanData span) { + String outputJson = + span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); + assertNotNull(outputJson, "span braintrust.output_json must be set"); + return JSON_MAPPER.readTree(outputJson); + } + @SneakyThrows private static void assertValidOpenAISpan(SpanData span, boolean isStreaming) { var attributes = span.getAttributes(); diff --git a/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java b/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java index 21bde78a..7e9a7833 100644 --- a/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java +++ b/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java @@ -1,5 +1,7 @@ package dev.braintrust.instrumentation.springai.v1_0_0; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import dev.braintrust.instrumentation.InstrumentationSemConv; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Span; @@ -12,6 +14,7 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -553,82 +556,24 @@ static String reassembleSSEResponse(String rawSSE, Span span, StreamContext stre @SneakyThrows private static String reassembleOpenAISSE(String rawSSE, Span span, StreamContext streamCtx) { var mapper = dev.braintrust.json.BraintrustJsonMapper.get(); - var choices = mapper.createObjectNode(); - var usage = mapper.createObjectNode(); - String model = null; - + // Generically merge chunks so every streamed field (content, reasoning, tool calls, ...) is + // reconstructed — the shared accumulator merges tool-call deltas by index, which a naive + // append-per-fragment loop cannot do. + var accumulator = new dev.braintrust.instrumentation.SseResponseAccumulator(mapper); for (String line : rawSSE.split("\n")) { - if (!line.startsWith("data: ") || line.equals("data: [DONE]")) { + if (!line.startsWith("data: ")) { continue; } - String json = line.substring("data: ".length()).trim(); - var chunk = mapper.readTree(json); - - if (model == null && chunk.has("model") && !chunk.get("model").isNull()) { - model = chunk.get("model").asText(); - } - - if (chunk.has("choices")) { - for (var choiceChunk : chunk.get("choices")) { - int index = choiceChunk.has("index") ? choiceChunk.get("index").asInt() : 0; - String indexKey = String.valueOf(index); - - if (!choices.has(indexKey)) { - var choice = mapper.createObjectNode(); - var message = mapper.createObjectNode(); - message.put("role", "assistant"); - message.put("content", ""); - choice.set("message", message); - choice.put("index", index); - choices.set(indexKey, choice); - } - - var choice = choices.get(indexKey); - if (choiceChunk.has("delta")) { - var delta = choiceChunk.get("delta"); - if (delta.has("content") && !delta.get("content").isNull()) { - String existing = choice.get("message").get("content").asText(); - ((com.fasterxml.jackson.databind.node.ObjectNode) choice.get("message")) - .put("content", existing + delta.get("content").asText()); - } - if (delta.has("tool_calls")) { - if (!choice.get("message").has("tool_calls")) { - ((com.fasterxml.jackson.databind.node.ObjectNode) - choice.get("message")) - .set("tool_calls", mapper.createArrayNode()); - } - for (var tc : delta.get("tool_calls")) { - ((com.fasterxml.jackson.databind.node.ArrayNode) - choice.get("message").get("tool_calls")) - .add(tc); - } - } - } - if (choiceChunk.has("finish_reason") - && !choiceChunk.get("finish_reason").isNull()) { - ((com.fasterxml.jackson.databind.node.ObjectNode) choice) - .put("finish_reason", choiceChunk.get("finish_reason").asText()); - } - } - } - - if (chunk.has("usage") && !chunk.get("usage").isNull()) { - var u = chunk.get("usage"); - u.fields().forEachRemaining(entry -> usage.set(entry.getKey(), entry.getValue())); - } + // merge() ignores the "[DONE]" sentinel and blank/non-JSON payloads. + accumulator.merge(line.substring("data: ".length())); } + String result = accumulator.build(); + var resultNode = mapper.readTree(result); + String model = resultNode.has("model") ? resultNode.get("model").asText() : null; backfillModelMetadata(span, streamCtx, model); - var choicesArray = mapper.createArrayNode(); - choices.fields().forEachRemaining(entry -> choicesArray.add(entry.getValue())); - - var result = mapper.createObjectNode(); - result.set("choices", choicesArray); - if (usage.size() > 0) { - result.set("usage", usage); - } - return dev.braintrust.json.BraintrustJsonMapper.toJson(result); + return result; } // ------------------------------------------------------------------------- @@ -636,73 +581,89 @@ private static String reassembleOpenAISSE(String rawSSE, Span span, StreamContex // ------------------------------------------------------------------------- /** - * Reassembles an Anthropic-format SSE stream into the Anthropic non-streaming response format: - * {@code {"role":"assistant","content":[{"type":"text","text":"..."}],"usage":{...}}} + * Reassembles an Anthropic-format SSE stream into the Anthropic non-streaming response shape: + * {@code {"role":"assistant","content":[],"usage":{...}}}. * *

Anthropic SSE events use types like {@code message_start}, {@code content_block_start}, - * {@code content_block_delta}, {@code message_delta}, and {@code message_stop}. + * {@code content_block_delta}, {@code message_delta}, and {@code message_stop}. Each content + * block is reconstructed by index, preserving its real type — {@code text}, {@code tool_use}, + * {@code thinking}, etc. — rather than collapsing everything to text: {@code text_delta} + * appends to a text block, {@code thinking_delta}/{@code signature_delta} to a thinking block, + * and {@code input_json_delta} fragments accumulate into a tool_use block's {@code input}. */ @SneakyThrows private static String reassembleAnthropicSSE( String rawSSE, Span span, StreamContext streamCtx) { var mapper = dev.braintrust.json.BraintrustJsonMapper.get(); - var contentBlocks = mapper.createArrayNode(); var usage = mapper.createObjectNode(); String model = null; + String stopReason = null; - // Track content blocks by index for delta merging - Map textByIndex = new HashMap<>(); + // Reconstruct each content block by index, preserving order and type. + Map blocksByIndex = new LinkedHashMap<>(); + // tool_use "input" streams as concatenated partial_json fragments; buffer per index and + // parse into an object at the end. + Map toolJsonByIndex = new HashMap<>(); for (String line : rawSSE.split("\n")) { - if (!line.startsWith("data: ") || line.equals("data: [DONE]")) { + if (!line.startsWith("data: ")) { continue; } String json = line.substring("data: ".length()).trim(); + if (json.isEmpty() || json.equals("[DONE]")) { + continue; + } var event = mapper.readTree(json); String type = event.has("type") ? event.get("type").asText() : ""; switch (type) { case "message_start" -> { - if (event.has("message")) { - var message = event.get("message"); - if (model == null - && message.has("model") - && !message.get("model").isNull()) { + var message = event.get("message"); + if (message != null && message.isObject()) { + if (model == null && message.hasNonNull("model")) { model = message.get("model").asText(); } - if (message.has("usage") && !message.get("usage").isNull()) { - message.get("usage") - .fields() - .forEachRemaining( - entry -> usage.set(entry.getKey(), entry.getValue())); - } + copyFields(message.get("usage"), usage); } } case "content_block_start" -> { int index = event.has("index") ? event.get("index").asInt() : 0; - textByIndex.putIfAbsent(index, new StringBuilder()); - // If the content block has initial text, capture it - if (event.has("content_block") - && event.get("content_block").has("text") - && !event.get("content_block").get("text").asText().isEmpty()) { - textByIndex - .get(index) - .append(event.get("content_block").get("text").asText()); - } + // Seed the block from the start event so type/id/name are preserved. + ObjectNode block = + event.has("content_block") && event.get("content_block").isObject() + ? (ObjectNode) event.get("content_block").deepCopy() + : mapper.createObjectNode(); + blocksByIndex.put(index, block); } case "content_block_delta" -> { int index = event.has("index") ? event.get("index").asInt() : 0; - textByIndex.putIfAbsent(index, new StringBuilder()); - if (event.has("delta") && event.get("delta").has("text")) { - textByIndex.get(index).append(event.get("delta").get("text").asText()); + ObjectNode block = + blocksByIndex.computeIfAbsent(index, i -> mapper.createObjectNode()); + var delta = event.get("delta"); + String deltaType = + delta != null && delta.has("type") ? delta.get("type").asText() : ""; + switch (deltaType) { + case "text_delta" -> appendText(block, "text", delta.get("text")); + case "thinking_delta" -> + appendText(block, "thinking", delta.get("thinking")); + case "signature_delta" -> + appendText(block, "signature", delta.get("signature")); + case "input_json_delta" -> { + if (delta.hasNonNull("partial_json")) { + toolJsonByIndex + .computeIfAbsent(index, i -> new StringBuilder()) + .append(delta.get("partial_json").asText()); + } + } + default -> { + // unknown delta type — nothing to accumulate + } } } case "message_delta" -> { - if (event.has("usage") && !event.get("usage").isNull()) { - event.get("usage") - .fields() - .forEachRemaining( - entry -> usage.set(entry.getKey(), entry.getValue())); + copyFields(event.get("usage"), usage); + if (event.has("delta") && event.get("delta").hasNonNull("stop_reason")) { + stopReason = event.get("delta").get("stop_reason").asText(); } } default -> { @@ -713,26 +674,54 @@ private static String reassembleAnthropicSSE( backfillModelMetadata(span, streamCtx, model); - // Build content blocks array - textByIndex.entrySet().stream() + // Finalize tool_use inputs: parse the accumulated partial_json into the block's "input". + for (var entry : toolJsonByIndex.entrySet()) { + ObjectNode block = blocksByIndex.get(entry.getKey()); + if (block == null) { + continue; + } + String jsonStr = entry.getValue().toString(); + block.set( + "input", + jsonStr.isEmpty() ? mapper.createObjectNode() : mapper.readTree(jsonStr)); + } + + var contentBlocks = mapper.createArrayNode(); + blocksByIndex.entrySet().stream() .sorted(Map.Entry.comparingByKey()) - .forEach( - entry -> { - var block = mapper.createObjectNode(); - block.put("type", "text"); - block.put("text", entry.getValue().toString()); - contentBlocks.add(block); - }); + .forEach(entry -> contentBlocks.add(entry.getValue())); var result = mapper.createObjectNode(); result.put("role", "assistant"); result.set("content", contentBlocks); + if (stopReason != null) { + result.put("stop_reason", stopReason); + } if (usage.size() > 0) { result.set("usage", usage); } return dev.braintrust.json.BraintrustJsonMapper.toJson(result); } + /** Appends a streamed textual delta onto {@code field}, seeding from empty if absent. */ + private static void appendText(ObjectNode block, String field, JsonNode value) { + if (value == null || value.isNull()) { + return; + } + String existing = block.has(field) ? block.get(field).asText() : ""; + block.put(field, existing + value.asText()); + } + + /** + * Copies all fields of {@code source} (an object, e.g. Anthropic usage) into {@code target}. + */ + private static void copyFields(JsonNode source, ObjectNode target) { + if (source == null || !source.isObject()) { + return; + } + source.fields().forEachRemaining(entry -> target.set(entry.getKey(), entry.getValue())); + } + // ------------------------------------------------------------------------- // Shared SSE helpers // ------------------------------------------------------------------------- diff --git a/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIAnthropicInstrumentationModule.java b/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIAnthropicInstrumentationModule.java index 358c97d1..72924636 100644 --- a/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIAnthropicInstrumentationModule.java +++ b/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIAnthropicInstrumentationModule.java @@ -32,6 +32,7 @@ public List getHelperClassNames() { PACKAGE + "BraintrustSpringAI$BodyCapturingRequest", PACKAGE + "BraintrustSpringAI$BodyCapturingOutputMessage", PACKAGE + "BraintrustSpringAI$StreamContext", + "dev.braintrust.instrumentation.SseResponseAccumulator", "dev.braintrust.json.BraintrustJsonMapper", "dev.braintrust.instrumentation.InstrumentationSemConv"); } diff --git a/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIOpenAIInstrumentationModule.java b/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIOpenAIInstrumentationModule.java index 60c13807..9b5ded17 100644 --- a/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIOpenAIInstrumentationModule.java +++ b/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIOpenAIInstrumentationModule.java @@ -32,6 +32,7 @@ public List getHelperClassNames() { PACKAGE + "BraintrustSpringAI$BodyCapturingRequest", PACKAGE + "BraintrustSpringAI$BodyCapturingOutputMessage", PACKAGE + "BraintrustSpringAI$StreamContext", + "dev.braintrust.instrumentation.SseResponseAccumulator", "dev.braintrust.json.BraintrustJsonMapper", "dev.braintrust.instrumentation.InstrumentationSemConv"); } diff --git a/braintrust-sdk/instrumentation/springai_1_0_0/src/test/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAITest.java b/braintrust-sdk/instrumentation/springai_1_0_0/src/test/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAITest.java index c56b53df..60bd07a8 100644 --- a/braintrust-sdk/instrumentation/springai_1_0_0/src/test/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAITest.java +++ b/braintrust-sdk/instrumentation/springai_1_0_0/src/test/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAITest.java @@ -27,6 +27,7 @@ import org.springframework.ai.openai.OpenAiChatModel; import org.springframework.ai.openai.OpenAiChatOptions; import org.springframework.ai.openai.api.OpenAiApi; +import org.springframework.ai.tool.function.FunctionToolCallback; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class BraintrustSpringAITest { @@ -225,6 +226,141 @@ && metrics(span).get("time_to_first_token").asLong() >= 0, "streaming responses should capture time to first token"); } + /** Tool-call parameter type; its shape drives the generated JSON schema. */ + record WeatherRequest(String location) {} + + /** + * OpenAI-only: a streaming call that provokes a tool call. The OpenAI streaming path routes + * through {@code reassembleOpenAISSE} → the shared {@link + * dev.braintrust.instrumentation.SseResponseAccumulator}, which merges the streamed tool-call + * argument fragments by index. {@code internalToolExecutionEnabled(false)} makes Spring return + * the tool call instead of executing it, so there is exactly one llm span to inspect. + */ + @org.junit.jupiter.api.Test + @SneakyThrows + void testStreamWithTools() { + Provider provider = + new Provider("openai", "openai", "gpt-4o-mini", TestHarness::openAiBaseUrl, true); + ChatModel chatModel = buildChatModel(provider); + + var weatherTool = + FunctionToolCallback.builder("get_weather", (WeatherRequest req) -> "unused") + .description("Get the current weather for a location") + .inputType(WeatherRequest.class) + .build(); + + var prompt = + new Prompt( + "What is the weather in Paris, France?", + OpenAiChatOptions.builder() + .model("gpt-4o-mini") + .temperature(0.0) + .maxTokens(256) + .streamUsage(true) + .toolCallbacks(weatherTool) + .internalToolExecutionEnabled(false) + .build()); + + boolean[] streamSurfacedToolCall = {false}; + chatModel.stream(prompt) + .doOnNext( + chunk -> { + var result = chunk.getResult(); + if (result != null + && result.getOutput() != null + && result.getOutput().hasToolCalls()) { + for (var toolCall : result.getOutput().getToolCalls()) { + if ("get_weather".equals(toolCall.name())) { + streamSurfacedToolCall[0] = true; + } + } + } + }) + .blockLast(); + assertTrue(streamSurfacedToolCall[0], "stream should surface a get_weather tool call"); + + var spans = testHarness.awaitExportedSpans(1); + assertEquals(1, spans.size()); + SpanData span = spans.get(0); + assertCommonSpanAttributes(span, provider); + + // The reconstructed streaming span output must carry the tool call, with its argument + // fragments merged into valid JSON — the bug this module previously had dropped them. + String outputJson = + span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); + assertNotNull(outputJson, "braintrust.output_json should be set"); + JsonNode toolCalls = + JSON_MAPPER.readTree(outputJson).get(0).get("message").get("tool_calls"); + assertNotNull(toolCalls, "streaming span output should contain tool_calls"); + assertEquals(1, toolCalls.size(), "expected a single tool call"); + JsonNode function = toolCalls.get(0).get("function"); + assertEquals("get_weather", function.get("name").asText(), "tool name should match"); + assertTrue( + JSON_MAPPER.readTree(function.get("arguments").asText()).has("location"), + "merged tool arguments should be valid JSON containing 'location'"); + } + + /** + * Anthropic-only: a streaming call that provokes a tool call. The Anthropic streaming path + * routes through {@code reassembleAnthropicSSE}, which must preserve the {@code tool_use} block + * — accumulating the streamed {@code input_json_delta} fragments into the block's {@code input} + * — rather than dropping everything but text. {@code internalToolExecutionEnabled(false)} keeps + * it to a single llm span. + * + *

(Extended thinking is not exercised here: Spring AI 1.0.0's own streaming deserializer + * only knows the {@code text}/{@code tool_use} content-block types and throws on {@code + * thinking} blocks, so a thinking stream can't complete through this client. The reassembly + * handles {@code thinking_delta} regardless, for clients/versions that do emit it.) + */ + @org.junit.jupiter.api.Test + @SneakyThrows + void testAnthropicStreamWithTools() { + Provider provider = + new Provider( + "anthropic", "anthropic", TEST_MODEL, TestHarness::anthropicBaseUrl, false); + ChatModel chatModel = buildChatModel(provider); + + var weatherTool = + FunctionToolCallback.builder("get_weather", (WeatherRequest req) -> "unused") + .description("Get the current weather for a location") + .inputType(WeatherRequest.class) + .build(); + + var options = + AnthropicChatOptions.builder() + .model(TEST_MODEL) + .temperature(0.0) + .maxTokens(1024) + .toolCallbacks(weatherTool) + .internalToolExecutionEnabled(false) + .build(); + + chatModel.stream(new Prompt("What is the weather in Paris, France?", options)).blockLast(); + + var spans = testHarness.awaitExportedSpans(1); + assertEquals(1, spans.size()); + SpanData span = spans.get(0); + assertCommonSpanAttributes(span, provider); + + // The reconstructed streaming output must carry the tool_use block with its input JSON + // merged from input_json_delta fragments — previously dropped by the text-only reassembly. + String outputJson = + span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); + assertNotNull(outputJson, "braintrust.output_json should be set"); + JsonNode content = JSON_MAPPER.readTree(outputJson).get("content"); + assertNotNull(content, "anthropic output should have a content array"); + + JsonNode toolUse = null; + for (JsonNode block : content) { + if ("tool_use".equals(block.path("type").asText())) toolUse = block; + } + assertNotNull(toolUse, "streaming output should preserve the tool_use block"); + assertEquals("get_weather", toolUse.path("name").asText(), "tool name should match"); + assertTrue( + toolUse.path("input").has("location"), + "tool_use input should be reconstructed JSON containing 'location'"); + } + // ------------------------------------------------------------------------- // Shared assertion helpers // ------------------------------------------------------------------------- diff --git a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/SseResponseAccumulator.java b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/SseResponseAccumulator.java new file mode 100644 index 00000000..ac8bf8a8 --- /dev/null +++ b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/SseResponseAccumulator.java @@ -0,0 +1,170 @@ +package dev.braintrust.instrumentation; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import javax.annotation.concurrent.NotThreadSafe; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +/** Reconstructs a full OpenAI-style chat completion response from SSE chunks. */ +@Slf4j +@NotThreadSafe +public final class SseResponseAccumulator { + private final ObjectMapper jsonMapper; + private final ObjectNode responseRoot; + // Accumulated choices keyed by their "index" so multi-choice (n>1) streams merge correctly. + private final Map choicesByIndex = new LinkedHashMap<>(); + // Scalar identifiers that a stream sends once per entity (never as fragments). Merging these + // last-write-wins rather than concatenating guards against a provider re-sending one across + // deltas and corrupting it (e.g. an "id" becoming "call_Xcall_X"). Streamed text fields + // (content, reasoning_content, tool-call arguments) are deliberately absent so they accumulate. + private static final Set NEVER_CONCAT = + Set.of("id", "type", "role", "name", "index", "finish_reason", "model", "object"); + + public SseResponseAccumulator(ObjectMapper jsonMapper) { + this.jsonMapper = jsonMapper; + this.responseRoot = jsonMapper.createObjectNode(); + } + + /** + * Merge one SSE {@code data:} payload into the reconstructed response. Blank payloads, the + * {@code [DONE]} sentinel, non-JSON, and non-object chunks are ignored, so callers can forward + * every event without pre-filtering. + */ + public void merge(String jsonChunk) { + if (jsonChunk == null) return; + String data = jsonChunk.strip(); + if (data.isEmpty() || "[DONE]".equals(data)) return; + + JsonNode chunk; + try { + chunk = jsonMapper.readTree(data); + } catch (JsonProcessingException e) { + log.debug("Failed to parse SSE chunk: {}", data, e); + return; + } + if (chunk == null || !chunk.isObject()) return; + + var fields = chunk.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + String name = entry.getKey(); + JsonNode value = entry.getValue(); + if ("choices".equals(name)) { + if (value.isArray()) { + value.forEach(this::mergeChoice); + } + } else if (!value.isNull()) { + responseRoot.set(name, value); + } + } + } + + /** + * Serialize the reconstructed response, assembling accumulated choices in index order. Safe to + * call once the stream is complete. + */ + @SneakyThrows(JsonProcessingException.class) + public String build() { + var choicesArray = jsonMapper.createArrayNode(); + // Emit in index order — choicesByIndex is insertion-ordered, so an out-of-order (n>1) + // stream could otherwise serialize choices out of index order. + choicesByIndex.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> choicesArray.add(entry.getValue())); + responseRoot.set("choices", choicesArray); + return jsonMapper.writeValueAsString(responseRoot); + } + + private void mergeChoice(JsonNode choiceChunk) { + if (!choiceChunk.isObject()) return; + int index = choiceChunk.has("index") ? choiceChunk.get("index").asInt() : 0; + ObjectNode choice = + choicesByIndex.computeIfAbsent( + index, + i -> { + var node = jsonMapper.createObjectNode(); + node.put("index", i); + return node; + }); + // Streaming nests the message under "delta"; the reconstructed non-streaming shape the UI + // expects uses "message". Rename, then deepMerge the whole choice so every field merges + // with the right semantics — including logprobs, whose per-token entries stream one-per- + // chunk and must accumulate rather than last-write-win. finish_reason stays last-write-wins + // (it's in NEVER_CONCAT and arrives once). + ObjectNode normalized = choiceChunk.deepCopy(); + if (normalized.has("delta")) { + normalized.set("message", normalized.remove("delta")); + } + deepMerge(choice, normalized); + } + + /** + * Recursively merges {@code source} into {@code target}. Textual leaves are concatenated (so + * streamed content / reasoning / tool-call arguments accumulate), except {@link #NEVER_CONCAT} + * scalar identifiers which are last-write-wins. Nested objects are merged key-by-key, and + * arrays whose elements carry an {@code index} (e.g. {@code tool_calls}) are merged by that + * index. Other scalars are last-write-wins. + */ + private static void deepMerge(ObjectNode target, JsonNode source) { + if (!source.isObject()) return; + var fields = source.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + String name = entry.getKey(); + JsonNode value = entry.getValue(); + JsonNode existing = target.get(name); + if (value.isTextual() + && existing != null + && existing.isTextual() + && !NEVER_CONCAT.contains(name)) { + target.put(name, existing.asText() + value.asText()); + } else if (value.isObject()) { + if (existing != null && existing.isObject()) { + deepMerge((ObjectNode) existing, value); + } else { + target.set(name, value.deepCopy()); + } + } else if (value.isArray()) { + mergeArray(target, name, value); + } else if (!value.isNull()) { + target.set(name, value); + } + } + } + + private static void mergeArray(ObjectNode target, String name, JsonNode sourceArray) { + if (!(target.get(name) instanceof ArrayNode targetArray)) { + target.set(name, sourceArray.deepCopy()); + return; + } + for (JsonNode element : sourceArray) { + ObjectNode match = + element.isObject() && element.has("index") + ? findByIndex(targetArray, element.get("index").asInt()) + : null; + if (match != null) { + deepMerge(match, element); + } else { + targetArray.add(element.deepCopy()); + } + } + } + + private static ObjectNode findByIndex(ArrayNode array, int index) { + for (JsonNode candidate : array) { + if (candidate.isObject() + && candidate.has("index") + && candidate.get("index").asInt() == index) { + return (ObjectNode) candidate; + } + } + return null; + } +} diff --git a/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/SseResponseAccumulatorTest.java b/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/SseResponseAccumulatorTest.java new file mode 100644 index 00000000..984d2aa4 --- /dev/null +++ b/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/SseResponseAccumulatorTest.java @@ -0,0 +1,133 @@ +package dev.braintrust.instrumentation; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; + +/** + * Hermetic tests for the generic SSE chunk merge. These assert that streamed fields are + * reconstructed field-agnostically — in particular that reasoning/thinking is preserved alongside + * content, and that tool-call deltas merge correctly — without needing a live provider. + */ +class SseResponseAccumulatorTest { + + private static final ObjectMapper JSON = new ObjectMapper(); + + @SneakyThrows + private static JsonNode reconstruct(String... chunks) { + var acc = new SseResponseAccumulator(JSON); + for (String chunk : chunks) { + acc.merge(chunk); + } + return JSON.readTree(acc.build()); + } + + @Test + void concatenatesContentAndPreservesReasoning() { + // A DeepSeek/OpenRouter-style reasoning stream: reasoning_content arrives before content. + var response = + reconstruct( + "{\"id\":\"x\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"reasoning_content\":\"Let" + + " me\"}}]}", + "{\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"" + + " think.\"}}]}", + "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Paris\"}}]}", + "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"finish_reason\":\"stop\"}]}", + "{\"choices\":[],\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":7,\"total_tokens\":12,\"completion_tokens_details\":{\"reasoning_tokens\":4}}}"); + + assertEquals("x", response.get("id").asText()); + assertEquals("m", response.get("model").asText()); + + JsonNode choice = response.get("choices").get(0); + assertEquals(0, choice.get("index").asInt()); + assertEquals("stop", choice.get("finish_reason").asText()); + + JsonNode message = choice.get("message"); + assertEquals("assistant", message.get("role").asText()); + assertEquals("Paris.", message.get("content").asText()); + assertEquals( + "Let me think.", + message.get("reasoning_content").asText(), + "reasoning/thinking content must survive reconstruction"); + + // usage passes through untouched so the semconv tool can map reasoning_tokens. + assertEquals( + 4, + response.get("usage") + .get("completion_tokens_details") + .get("reasoning_tokens") + .asInt()); + } + + @Test + void mergesToolCallDeltasByIndex() { + // Tool-call arguments are streamed in fragments and keyed by index; a naive + // content-only accumulator would drop these entirely. + var response = + reconstruct( + "{\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"getWeather\",\"arguments\":\"{\\\"loc\\\":\"}}]}}]}", + "{\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Paris\\\"}\"}}]}}]}", + "{\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}"); + + JsonNode toolCall = response.get("choices").get(0).get("message").get("tool_calls").get(0); + assertEquals("call_1", toolCall.get("id").asText()); + assertEquals("getWeather", toolCall.get("function").get("name").asText()); + assertEquals("{\"loc\":\"Paris\"}", toolCall.get("function").get("arguments").asText()); + } + + @Test + void concatenatesLogprobsAcrossChunks() { + // logprobs stream one token entry per chunk at the choice level (siblings of "delta"), with + // no per-element "index". Last-write-wins would keep only the final chunk's token, so the + // entries must accumulate across chunks. + var response = + reconstruct( + "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hi\"},\"logprobs\":{\"content\":[{\"token\":\"Hi\",\"logprob\":-0.1}]}}]}", + "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":{\"content\":[{\"token\":\"!\",\"logprob\":-0.2}]},\"finish_reason\":\"stop\"}]}"); + + JsonNode choice = response.get("choices").get(0); + assertEquals("Hi!", choice.get("message").get("content").asText()); + assertEquals("stop", choice.get("finish_reason").asText()); + + JsonNode logprobs = choice.get("logprobs").get("content"); + assertEquals(2, logprobs.size(), "logprobs entries from every chunk must be preserved"); + assertEquals("Hi", logprobs.get(0).get("token").asText()); + assertEquals("!", logprobs.get(1).get("token").asText()); + } + + @Test + void keepsMultipleChoicesSeparate() { + var response = + reconstruct( + "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"A\"}},{\"index\":1,\"delta\":{\"content\":\"B\"}}]}", + "{\"choices\":[{\"index\":1,\"delta\":{\"content\":\"B2\"}},{\"index\":0,\"delta\":{\"content\":\"A2\"}}]}"); + + assertEquals(2, response.get("choices").size()); + assertEquals("AA2", response.get("choices").get(0).get("message").get("content").asText()); + assertEquals("BB2", response.get("choices").get(1).get("message").get("content").asText()); + } + + @Test + void serializesChoicesInIndexOrderRegardlessOfArrival() { + // A provider could introduce a higher index before a lower one across chunks. Choices are + // keyed by index, so accumulation is unaffected, but the serialized array must still come + // out in index order (not first-seen order) so positional consumers see choices[i] == i. + var response = + reconstruct( + "{\"choices\":[{\"index\":2,\"delta\":{\"content\":\"C\"}}]}", + "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"A\"}}]}", + "{\"choices\":[{\"index\":1,\"delta\":{\"content\":\"B\"}}]}"); + + JsonNode choices = response.get("choices"); + assertEquals(3, choices.size()); + for (int i = 0; i < 3; i++) { + assertEquals(i, choices.get(i).get("index").asInt(), "choices must be in index order"); + } + assertEquals("A", choices.get(0).get("message").get("content").asText()); + assertEquals("B", choices.get(1).get("message").get("content").asText()); + assertEquals("C", choices.get(2).get("message").get("content").asText()); + } +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-538b5ad9c2fe.txt b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-538b5ad9c2fe.txt new file mode 100644 index 00000000..35c4b29b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-538b5ad9c2fe.txt @@ -0,0 +1,22 @@ +data: {"id":"chatcmpl-ECHRyVCdHnM9OEpFBhFG8RzmndiYh","object":"chat.completion.chunk","created":1786595722,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_17e3c4f467","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_YzrsLLgDExH6lDeyfyLGZfzf","type":"function","function":{"name":"get_weather","arguments":""}}],"refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"24lDEkHwG"} + +data: {"id":"chatcmpl-ECHRyVCdHnM9OEpFBhFG8RzmndiYh","object":"chat.completion.chunk","created":1786595722,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_17e3c4f467","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"iRR"} + +data: {"id":"chatcmpl-ECHRyVCdHnM9OEpFBhFG8RzmndiYh","object":"chat.completion.chunk","created":1786595722,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_17e3c4f467","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"location"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"PH81B2RXCELeVy"} + +data: {"id":"chatcmpl-ECHRyVCdHnM9OEpFBhFG8RzmndiYh","object":"chat.completion.chunk","created":1786595722,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_17e3c4f467","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"r"} + +data: {"id":"chatcmpl-ECHRyVCdHnM9OEpFBhFG8RzmndiYh","object":"chat.completion.chunk","created":1786595722,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_17e3c4f467","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"Paris"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"U"} + +data: {"id":"chatcmpl-ECHRyVCdHnM9OEpFBhFG8RzmndiYh","object":"chat.completion.chunk","created":1786595722,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_17e3c4f467","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":","}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"na9eB"} + +data: {"id":"chatcmpl-ECHRyVCdHnM9OEpFBhFG8RzmndiYh","object":"chat.completion.chunk","created":1786595722,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_17e3c4f467","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" France"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"t67urqAfYzw4YkA"} + +data: {"id":"chatcmpl-ECHRyVCdHnM9OEpFBhFG8RzmndiYh","object":"chat.completion.chunk","created":1786595722,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_17e3c4f467","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"GnF"} + +data: {"id":"chatcmpl-ECHRyVCdHnM9OEpFBhFG8RzmndiYh","object":"chat.completion.chunk","created":1786595722,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_17e3c4f467","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":null,"obfuscation":"fzoa"} + +data: {"id":"chatcmpl-ECHRyVCdHnM9OEpFBhFG8RzmndiYh","object":"chat.completion.chunk","created":1786595722,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_17e3c4f467","choices":[],"usage":{"prompt_tokens":60,"completion_tokens":16,"total_tokens":76,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"w4h2BjdLR6FblLt"} + +data: [DONE] + diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-acc477a295ad.txt b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-acc477a295ad.txt new file mode 100644 index 00000000..829f2b23 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-acc477a295ad.txt @@ -0,0 +1,22 @@ +data: {"id":"chatcmpl-ECHGLaNOeixRIOA5U84vJQBxpnmLO","object":"chat.completion.chunk","created":1786595001,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_64d0f9e03c","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_aHe1Xt2XJOAczfytVQ5tnL44","type":"function","function":{"name":"get_weather","arguments":""}}],"refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"YRAAKHahg"} + +data: {"id":"chatcmpl-ECHGLaNOeixRIOA5U84vJQBxpnmLO","object":"chat.completion.chunk","created":1786595001,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_64d0f9e03c","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"gOk"} + +data: {"id":"chatcmpl-ECHGLaNOeixRIOA5U84vJQBxpnmLO","object":"chat.completion.chunk","created":1786595001,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_64d0f9e03c","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"location"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"DbG9qMhcoee04s"} + +data: {"id":"chatcmpl-ECHGLaNOeixRIOA5U84vJQBxpnmLO","object":"chat.completion.chunk","created":1786595001,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_64d0f9e03c","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"m"} + +data: {"id":"chatcmpl-ECHGLaNOeixRIOA5U84vJQBxpnmLO","object":"chat.completion.chunk","created":1786595001,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_64d0f9e03c","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"Paris"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"w"} + +data: {"id":"chatcmpl-ECHGLaNOeixRIOA5U84vJQBxpnmLO","object":"chat.completion.chunk","created":1786595001,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_64d0f9e03c","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":","}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"zkpFV"} + +data: {"id":"chatcmpl-ECHGLaNOeixRIOA5U84vJQBxpnmLO","object":"chat.completion.chunk","created":1786595001,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_64d0f9e03c","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" France"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"MycfO95hXiKPUw7"} + +data: {"id":"chatcmpl-ECHGLaNOeixRIOA5U84vJQBxpnmLO","object":"chat.completion.chunk","created":1786595001,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_64d0f9e03c","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"MjJ"} + +data: {"id":"chatcmpl-ECHGLaNOeixRIOA5U84vJQBxpnmLO","object":"chat.completion.chunk","created":1786595001,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_64d0f9e03c","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":null,"obfuscation":"jaUI"} + +data: {"id":"chatcmpl-ECHGLaNOeixRIOA5U84vJQBxpnmLO","object":"chat.completion.chunk","created":1786595001,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_64d0f9e03c","choices":[],"usage":{"prompt_tokens":68,"completion_tokens":16,"total_tokens":84,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"uMc9Q5wngKW0B0S"} + +data: [DONE] + diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-9a580e94f540.txt b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-9a580e94f540.txt new file mode 100644 index 00000000..fb91f3cc --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-9a580e94f540.txt @@ -0,0 +1,39 @@ +event: response.created +data: {"type":"response.created","response":{"id":"resp_026c45b08e65b9f1006a7d498ce180819aaf91ff17efa5cc3e","object":"response","created_at":1786595724,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Get the current weather for a location","name":"get_weather","output_schema":null,"parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state"}},"required":["location"]},"strict":false}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + +event: response.in_progress +data: {"type":"response.in_progress","response":{"id":"resp_026c45b08e65b9f1006a7d498ce180819aaf91ff17efa5cc3e","object":"response","created_at":1786595724,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Get the current weather for a location","name":"get_weather","output_schema":null,"parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state"}},"required":["location"]},"strict":false}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","type":"function_call","status":"in_progress","arguments":"","call_id":"call_42HmCE0PN6HC6lCVKtRRUdU7","name":"get_weather"},"output_index":0,"sequence_number":2} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","obfuscation":"qJHhtBjqWgwkEj","output_index":0,"sequence_number":3} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"location","item_id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","obfuscation":"uxd3JppC","output_index":0,"sequence_number":4} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"\":\"","item_id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","obfuscation":"ymfMrvVzWDJHj","output_index":0,"sequence_number":5} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"Paris","item_id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","obfuscation":"0IaHOmRA2No","output_index":0,"sequence_number":6} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":",","item_id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","obfuscation":"veiRGFOE11ADrRd","output_index":0,"sequence_number":7} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":" France","item_id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","obfuscation":"zJF1wnhF1","output_index":0,"sequence_number":8} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"\"}","item_id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","obfuscation":"Fz2fYjJ33OVnhC","output_index":0,"sequence_number":9} + +event: response.function_call_arguments.done +data: {"type":"response.function_call_arguments.done","arguments":"{\"location\":\"Paris, France\"}","item_id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","output_index":0,"sequence_number":10} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","type":"function_call","status":"completed","arguments":"{\"location\":\"Paris, France\"}","call_id":"call_42HmCE0PN6HC6lCVKtRRUdU7","name":"get_weather"},"output_index":0,"sequence_number":11} + +event: response.completed +data: {"type":"response.completed","response":{"id":"resp_026c45b08e65b9f1006a7d498ce180819aaf91ff17efa5cc3e","object":"response","created_at":1786595724,"status":"completed","background":false,"completed_at":1786595725,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[{"id":"fc_026c45b08e65b9f1006a7d498ddbd8819aaa4874bb04c1e2d7","type":"function_call","status":"completed","arguments":"{\"location\":\"Paris, France\"}","call_id":"call_42HmCE0PN6HC6lCVKtRRUdU7","name":"get_weather"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Get the current weather for a location","name":"get_weather","output_schema":null,"parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state"}},"required":["location"]},"strict":false}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":54,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":17,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":71},"user":null,"metadata":{}},"sequence_number":12} + diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-538b5ad9c2fe.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-538b5ad9c2fe.json new file mode 100644 index 00000000..bf4ca918 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-538b5ad9c2fe.json @@ -0,0 +1,49 @@ +{ + "id" : "0b3f949e-8eed-3c3a-ba30-2d5a34244a85", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"What is the weather in Paris, France?\",\"role\":\"user\"}],\"model\":\"gpt-4o\",\"stream_options\":{\"include_usage\":true},\"temperature\":0.0,\"tools\":[{\"function\":{\"name\":\"get_weather\",\"description\":\"Get the current weather for a location\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and state\"}},\"required\":[\"location\"]}},\"type\":\"function\"}],\"stream\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-538b5ad9c2fe.txt", + "headers" : { + "x-request-id" : "req_13f4a217fdd241549b5fbe80a1ca5a91", + "x-ratelimit-limit-tokens" : "30000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2a50341f8c5ec58-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "6ms", + "x-ratelimit-remaining-tokens" : "29999988", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "9999", + "Date" : "Thu, 13 Aug 2026 04:35:23 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=Mj8SAhrr9xv2LQBicpHe_QH_si4Kknu50jwS0PnVLCo-1786595722.5553038-1.0.1.1-8xrLz77gnB8Djvjs3t1_BfqTUx1Z7v4cuC2_1jlIURCSXRm6rPWpQynT6sfdmQLidraqbT6OcACWit951BRdPNys9AJ_8gFcwfppzoA.tJ0mAo17rJCalOolGThS7BwS; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 13 Aug 2026 05:05:23 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "10000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "442", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "text/event-stream; charset=utf-8" + } + }, + "uuid" : "0b3f949e-8eed-3c3a-ba30-2d5a34244a85", + "persistent" : true, + "insertionIndex" : 34 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-acc477a295ad.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-acc477a295ad.json new file mode 100644 index 00000000..86bf7af8 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-acc477a295ad.json @@ -0,0 +1,49 @@ +{ + "id" : "b401ee59-4b78-3f00-aaf8-7bed0d52cbad", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"gpt-4o\",\n \"messages\" : [ {\n \"role\" : \"user\",\n \"content\" : \"What is the weather in Paris, France?\"\n } ],\n \"temperature\" : 0.0,\n \"stream\" : true,\n \"stream_options\" : {\n \"include_usage\" : true\n },\n \"tools\" : [ {\n \"type\" : \"function\",\n \"function\" : {\n \"name\" : \"get_weather\",\n \"description\" : \"Get the current weather for a location\",\n \"parameters\" : {\n \"type\" : \"object\",\n \"properties\" : {\n \"location\" : {\n \"type\" : \"string\",\n \"description\" : \"The city and state, e.g. San Francisco, CA\"\n }\n },\n \"required\" : [ \"location\" ]\n }\n }\n } ]\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-acc477a295ad.txt", + "headers" : { + "x-request-id" : "req_86452c065dc04d92acb7736fd94dea30", + "x-ratelimit-limit-tokens" : "30000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2a4f19e59cf76a8-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "6ms", + "x-ratelimit-remaining-tokens" : "29999988", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "9999", + "Date" : "Thu, 13 Aug 2026 04:23:21 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=iKlivIn4kPvGXz5ZvduibLCZ_B.xcNxDXE05J.Qyn0I-1786595000.0603495-1.0.1.1-eldXPa9Oseu2hVMGvC87huKrVDaMXnFa9a6znT3sf8EnxIr2Z.HhFtnzmYXmw9ach0XPDw6Wr4MojCaAJPQ0iYfDIVIwOpUwzQUC.pzYCwUeJtat9S3.Vcj83gozyX6u; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 13 Aug 2026 04:53:21 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "10000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "510", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "text/event-stream; charset=utf-8" + } + }, + "uuid" : "b401ee59-4b78-3f00-aaf8-7bed0d52cbad", + "persistent" : true, + "insertionIndex" : 32 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-9a580e94f540.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-9a580e94f540.json new file mode 100644 index 00000000..f33df5ab --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-9a580e94f540.json @@ -0,0 +1,42 @@ +{ + "id" : "229a99a3-e268-3598-8770-2734af5feb68", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":[{\"content\":\"What is the weather in Paris, France?\",\"role\":\"user\"}],\"model\":\"gpt-4o\",\"tools\":[{\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and state\"}},\"required\":[\"location\"]},\"strict\":false,\"type\":\"function\",\"description\":\"Get the current weather for a location\"}],\"stream\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-9a580e94f540.txt", + "headers" : { + "x-request-id" : "req_3d3be76684d840438a78d39d1d84d46f", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2a5034bedaaed00-SEA", + "X-Content-Type-Options" : "nosniff", + "Date" : "Thu, 13 Aug 2026 04:35:25 GMT", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=63SwArwxiYRdhAVqzmJPPxf.qx3GXWGTr6wl7TSDKfU-1786595724.1437151-1.0.1.1-awf4xLyqWuD_Nws.iOBD_85ygIt5LhVDrg7rkK8Gic0S_Oz1TvMHIlmVHVlPCcBhoEJ32QmfCixzDZYVvYsHFtfjTEgBqCdsTfz1VvRMk9sdPJsAC4YP.30qzWXbyHfq; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 13 Aug 2026 05:05:25 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "203", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "text/event-stream; charset=utf-8" + } + }, + "uuid" : "229a99a3-e268-3598-8770-2734af5feb68", + "persistent" : true, + "insertionIndex" : 33 +} \ No newline at end of file