From ce5b7b374b1225a8dc005ec1eb3aae04be7199af Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Thu, 13 Aug 2026 22:49:51 -0600 Subject: [PATCH 1/3] VCR --- .../chat_completions-7c0db28da6b8.json | 35 +++++++++++++ .../chat_completions-b5bcb38ce8df.json | 35 +++++++++++++ .../chat_completions-7c0db28da6b8.json | 49 +++++++++++++++++++ .../chat_completions-b5bcb38ce8df.json | 49 +++++++++++++++++++ 4 files changed, 168 insertions(+) create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-7c0db28da6b8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-b5bcb38ce8df.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-7c0db28da6b8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-b5bcb38ce8df.json diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-7c0db28da6b8.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-7c0db28da6b8.json new file mode 100644 index 00000000..8f664dfa --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-7c0db28da6b8.json @@ -0,0 +1,35 @@ +{ + "id": "chatcmpl-ECe43IuPfU77adv4loq6PNdopaMx2", + "object": "chat.completion", + "created": 1786682651, + "model": "o4-mini-2025-04-16", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Each term is the product of two consecutive integers. In fact\n\n a₁=1·2=2, \n a₂=2·3=6, \n a₃=3·4=12, \n a₄=4·5=20, \n a₅=5·6=30, …\n\nso for n≥1\n\n aₙ = n·(n+1)\n\nEquivalently aₙ = n² + n.", + "refusal": null, + "annotations": [] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 41, + "completion_tokens": 826, + "total_tokens": 867, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 704, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": null +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-b5bcb38ce8df.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-b5bcb38ce8df.json new file mode 100644 index 00000000..2e872a27 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-b5bcb38ce8df.json @@ -0,0 +1,35 @@ +{ + "id": "chatcmpl-ECe43NIjWyAmKELxrwXrmNLta5t8z", + "object": "chat.completion", + "created": 1786682651, + "model": "o4-mini-2025-04-16", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Each term is the product of two consecutive positive integers:\n\nTerm 1: 1×2 = 2 \nTerm 2: 2×3 = 6 \nTerm 3: 3×4 = 12 \nTerm 4: 4×5 = 20 \nTerm 5: 5×6 = 30 \n\nIf we call the nth term aₙ (with n starting at 1), then\n\n aₙ = n·(n + 1)\n\nEquivalently,\n\n aₙ = n² + n.", + "refusal": null, + "annotations": [] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 41, + "completion_tokens": 525, + "total_tokens": 566, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 384, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": null +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-7c0db28da6b8.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-7c0db28da6b8.json new file mode 100644 index 00000000..10c57661 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-7c0db28da6b8.json @@ -0,0 +1,49 @@ +{ + "id" : "a17f0a08-90cf-3039-a216-15f16dbeedf0", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what would be the formula for the nth term?\\n\",\"role\":\"user\"}],\"model\":\"o4-mini\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-7c0db28da6b8.json", + "headers" : { + "Server" : "cloudflare", + "x-ratelimit-reset-tokens" : "0s", + "set-cookie" : "__cf_bm=Wky4DpVpTg7LMCTKr3ZOE9ZrY3w_.7Zx_.tQJ_dwUkI-1786682651.1902754-1.0.1.1-_5dU7Ifp7l3K1Eg1dXMsO63q0vFVHk2LQSqJZkQrbml3gWFkrwA7TGO6kZkUNJJIOq1U0axg3NrBUO3YO3w_ZWDP0Uy.LNbn9TcHHkC0gbI8jpuhRxDt_rJ5aJQRQy2y; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 14 Aug 2026 05:14:18 GMT", + "Access-Control-Expose-Headers" : [ "CF-Ray", "X-Request-ID", "CF-Ray" ], + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json", + "x-request-id" : "req_3dc11519d9c8408abd0cb1c195d19d19", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "CF-Ray" : "a2ad4d89ee3f1f9a-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999970", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Fri, 14 Aug 2026 04:44:18 GMT", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "6333", + "alt-svc" : "h3=\":443\"; ma=86400" + } + }, + "uuid" : "a17f0a08-90cf-3039-a216-15f16dbeedf0", + "persistent" : true, + "insertionIndex" : 37 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-b5bcb38ce8df.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-b5bcb38ce8df.json new file mode 100644 index 00000000..84459ff3 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-b5bcb38ce8df.json @@ -0,0 +1,49 @@ +{ + "id" : "2c18cacf-35a1-374b-9f56-93255fa2c5ec", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"o4-mini\",\n \"messages\" : [ {\n \"role\" : \"user\",\n \"content\" : \"Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what would be the formula for the nth term?\\n\"\n } ],\n \"stream\" : false,\n \"reasoning_effort\" : \"medium\"\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-b5bcb38ce8df.json", + "headers" : { + "x-request-id" : "req_08731d3c4d734f9dbec8937ff43f6b9a", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2ad4d87c8531421-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999970", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Fri, 14 Aug 2026 04:44:16 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=a2csL43EaTjPoUL5rKW8RS3vqRVUbFvwoLqGuFUXFj8-1786682650.844532-1.0.1.1-yVMQBonLnMHpTtOz6AqSFb2krLVftOtfFEkpd2nsZoy7sHK04YLiL3rt7Bp1epw8TNp0HKYG3M4IRs4EADUzn5YO01gAdjaqU2MMGRZX7WXZLBHjvCpfe4_OzAqJZ.Vh; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 14 Aug 2026 05:14:16 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "4468", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "2c18cacf-35a1-374b-9f56-93255fa2c5ec", + "persistent" : true, + "insertionIndex" : 38 +} \ No newline at end of file From d570ae3dfa911c62138dbec73467e7aeabe79c7a Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Thu, 13 Aug 2026 22:49:35 -0600 Subject: [PATCH 2/3] bump spec version --- .../com/google/genai/BraintrustApiClient.java | 2 +- .../genai/v1_18_0/BraintrustGenAITest.java | 4 ++-- .../sdkspecimpl/SpecClientRegistry.java | 17 +++++++++++++++- .../sdkspecimpl/clients/GoogleSpecClient.java | 20 ++++++++++++++++++- gradle.properties | 2 +- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java b/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java index dc52db2d..1a108cdb 100644 --- a/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java +++ b/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java @@ -55,7 +55,7 @@ private void tagSpan( @Nullable String responseBody) { try { Map metadata = new java.util.HashMap<>(); - metadata.put("provider", "gemini"); + metadata.put("provider", "google"); // Parse request if (requestBody != null) { diff --git a/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/dev/braintrust/instrumentation/genai/v1_18_0/BraintrustGenAITest.java b/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/dev/braintrust/instrumentation/genai/v1_18_0/BraintrustGenAITest.java index 35b2c683..bc2e066e 100644 --- a/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/dev/braintrust/instrumentation/genai/v1_18_0/BraintrustGenAITest.java +++ b/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/dev/braintrust/instrumentation/genai/v1_18_0/BraintrustGenAITest.java @@ -67,7 +67,7 @@ void testWrapGemini() { span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata")); assertNotNull(metadataJson, "braintrust.metadata should be set"); var metadata = JSON_MAPPER.readTree(metadataJson); - assertEquals("gemini", metadata.get("provider").asText()); + assertEquals("google", metadata.get("provider").asText()); assertEquals(MODEL_ID, metadata.get("model").asText()); assertEquals(0.0, metadata.get("temperature").asDouble()); assertEquals(50, metadata.get("maxOutputTokens").asInt()); @@ -145,7 +145,7 @@ void testWrapGeminiAsync() { span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata")); assertNotNull(metadataJson, "braintrust.metadata should be set"); var metadata = JSON_MAPPER.readTree(metadataJson); - assertEquals("gemini", metadata.get("provider").asText()); + assertEquals("google", metadata.get("provider").asText()); assertEquals(MODEL_ID, metadata.get("model").asText()); assertEquals(0.0, metadata.get("temperature").asDouble()); assertEquals(50, metadata.get("maxOutputTokens").asInt()); diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java index ad3987e9..e5eebeb1 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java @@ -32,7 +32,22 @@ public final class SpecClientRegistry { * nothing instead of producing a failing "unsupported" test. Prefer registering a client; reach * for this only when no Java client can express the spec at all. */ - private static final Set KNOWN_UNSUPPORTED_SPECS = Set.of(); + private static final Set KNOWN_UNSUPPORTED_SPECS = + Set.of( + // Brand-new Google features added in braintrust-spec v0.0.10. The genai + // instrumentation / GoogleSpecClient don't implement them yet (thinking, + // grounding tools, response modalities, per-modality prompt-token extraction), + // so they're marked unsupported rather than half-implemented. GoogleSpecClient + // also filters these out of supports(); see its UNSUPPORTED_SPECS. + "google/thinking", + "google/grounding", + "google/streaming", + "google/generated_audio_usage", + "google/generated_image_usage", + "google/attachments", + // Google Interactions API (/v1/interactions) — no Java client exists. + "google/interactions", + "google/interactions_streaming"); private static final List CLIENTS = Stream.of( diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java index a2bb597e..8ff00341 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java @@ -17,6 +17,23 @@ /** Google Gemini (google-genai) client: generateContent (sync + streaming). */ public final class GoogleSpecClient implements SpecClient { + /** + * {@code :generateContent} specs that require Google features not yet implemented in the genai + * instrumentation / this client (thinking, grounding tools, response modalities, and + * per-modality prompt-token extraction for attachments). They are additionally listed in {@link + * dev.braintrust.sdkspecimpl.SpecClientRegistry#KNOWN_UNSUPPORTED_SPECS} so they load to a + * deliberate skip instead of a failing "unsupported" sentinel. Remove an entry here once the + * corresponding feature lands. + */ + private static final java.util.Set UNSUPPORTED_SPECS = + java.util.Set.of( + "thinking", + "grounding", + "streaming", + "generated_audio_usage", + "generated_image_usage", + "attachments"); + private volatile Client geminiClient; @Override @@ -31,7 +48,8 @@ public String provider() { @Override public boolean supports(LlmSpanSpec spec) { - return spec.endpoint().contains(":generateContent"); + return spec.endpoint().contains(":generateContent") + && !UNSUPPORTED_SPECS.contains(spec.name()); } @Override diff --git a/gradle.properties b/gradle.properties index 103fc769..52a608c5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,7 +8,7 @@ org.gradle.daemon=true org.gradle.warning.mode=summary # braintrust-spec git ref (SHA or tag) used by btx tests -braintrustSpecRef=v0.0.9 +braintrustSpecRef=v0.0.10 # braintrust-openapi commit SHA used by braintrust-api braintrustOpenApiRef=64b79cb9122f50a74eac98ea86c3ec1858c0cdd1 From 626684802918b2b2a8919d40b0dde41fd2246ccc Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Fri, 14 Aug 2026 20:48:06 -0600 Subject: [PATCH 3/3] wip --- .../openai/v2_15_0/TracingHttpClient.java | 36 +- .../openai/v2_15_0/BraintrustOpenAITest.java | 34 +- .../BraintrustOpenAIWebSearchTest.java | 143 ++++++++ .../instrumentation/ResponseToolSpans.java | 236 +++++++++++++ .../ResponseToolSpansTest.java | 224 +++++++++++++ .../__files/responses-0c37df81d2cf.json | 127 +++++++ .../openai/__files/responses-31d9a6eee624.txt | 315 ++++++++++++++++++ .../mappings/responses-0c37df81d2cf.json | 48 +++ .../mappings/responses-31d9a6eee624.json | 42 +++ 9 files changed, 1191 insertions(+), 14 deletions(-) create mode 100644 braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAIWebSearchTest.java create mode 100644 braintrust-sdk/src/main/java/dev/braintrust/instrumentation/ResponseToolSpans.java create mode 100644 braintrust-sdk/src/test/java/dev/braintrust/instrumentation/ResponseToolSpansTest.java create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-0c37df81d2cf.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-31d9a6eee624.txt create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-0c37df81d2cf.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-31d9a6eee624.json diff --git a/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java b/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java index 6f9fac29..cd76650b 100644 --- a/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java +++ b/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java @@ -10,6 +10,7 @@ import com.openai.models.responses.ResponseStreamEvent; import dev.braintrust.bootstrap.BraintrustBridge; import dev.braintrust.instrumentation.InstrumentationSemConv; +import dev.braintrust.instrumentation.ResponseToolSpans; import dev.braintrust.json.BraintrustJsonMapper; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Span; @@ -130,7 +131,7 @@ public void close() { var response = underlying.execute(bufferedRequest, requestOptions); // Always tee the response body. onStreamClosed() detects whether the collected // bytes are SSE or plain JSON and tags the span accordingly. - return new TeeingStreamHttpResponse(response, span); + return new TeeingStreamHttpResponse(response, span, tracer); } catch (Exception e) { InstrumentationSemConv.tagLLMSpanResponse(span, e); span.end(); @@ -159,7 +160,9 @@ public void close() { return underlying .executeAsync(bufferedRequest, requestOptions) .thenApply( - response -> (HttpResponse) new TeeingStreamHttpResponse(response, span)) + response -> + (HttpResponse) + new TeeingStreamHttpResponse(response, span, tracer)) .whenComplete( (response, t) -> { if (t != null) { @@ -239,21 +242,22 @@ private static String readBodyAsString(HttpRequestBody body) { * the bytes are an SSE stream (first non-empty line starts with {@code "data: "}) or a plain * JSON response, and parses accordingly. */ - private static void tagSpanFromBuffer(Span span, byte[] bytes, Long timeToFirstTokenNanos) { - if (bytes.length == 0) return; + private static String tagSpanFromBuffer(Span span, byte[] bytes, Long timeToFirstTokenNanos) { + if (bytes.length == 0) return null; try { String firstLine = firstNonEmptyLine(bytes); if (firstLine != null && (firstLine.startsWith("data:") || firstLine.startsWith("event:"))) { - tagSpanFromSseBytes(span, bytes, timeToFirstTokenNanos); + return tagSpanFromSseBytes(span, bytes, timeToFirstTokenNanos); } else { + String responseJson = new String(bytes, StandardCharsets.UTF_8); InstrumentationSemConv.tagLLMSpanResponse( - span, - InstrumentationSemConv.PROVIDER_NAME_OPENAI, - new String(bytes, StandardCharsets.UTF_8)); + span, InstrumentationSemConv.PROVIDER_NAME_OPENAI, responseJson); + return responseJson; } } catch (Exception e) { log.error("Could not tag span from response buffer", e); + return null; } } @@ -273,7 +277,7 @@ private static String firstNonEmptyLine(byte[] bytes) { * Parses SSE wire bytes, feeds each {@code data:} chunk through {@link * ChatCompletionAccumulator}, then tags the span with the reassembled output JSON. */ - private static void tagSpanFromSseBytes( + private static String tagSpanFromSseBytes( Span span, byte[] sseBytes, Long timeToFirstTokenNanos) { try { var reader = @@ -330,8 +334,10 @@ private static void tagSpanFromSseBytes( responseJson, timeToFirstTokenNanos); } + return responseJson; } catch (Exception e) { log.error("Could not parse SSE buffer to tag streaming span output", e); + return null; } } @@ -343,14 +349,16 @@ private static void tagSpanFromSseBytes( private static final class TeeingStreamHttpResponse implements HttpResponse { private final HttpResponse delegate; private final Span span; + private final Tracer tracer; private final long spanStartNanos = System.nanoTime(); private final AtomicLong timeToFirstTokenNanos = new AtomicLong(); private final ByteArrayOutputStream teeBuffer = new ByteArrayOutputStream(); private final InputStream teeStream; - TeeingStreamHttpResponse(HttpResponse delegate, Span span) { + TeeingStreamHttpResponse(HttpResponse delegate, Span span, Tracer tracer) { this.delegate = delegate; this.span = span; + this.tracer = tracer; this.teeStream = new TeeInputStream( delegate.body(), teeBuffer, this::onFirstByte, this::onStreamClosed); @@ -369,7 +377,13 @@ private void onStreamClosed() { synchronized (teeBuffer) { bytes = teeBuffer.toByteArray(); } - tagSpanFromBuffer(span, bytes, timeToFirstTokenNanos.get()); + String responseJson = tagSpanFromBuffer(span, bytes, timeToFirstTokenNanos.get()); + if (responseJson != null) { + // Emit child tool spans (web search, etc.) parented to the LLM span, while it + // is still live. No-op for Chat Completions responses (no `output` array). + ResponseToolSpans.emitOpenAIResponseToolSpans( + tracer, Context.current().with(span), responseJson); + } } finally { span.end(); } diff --git a/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java index b34e5fc7..2a96d73f 100644 --- a/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java +++ b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java @@ -545,10 +545,18 @@ void testResponsesStreamingWithTools() { .toList(); assertFalse(functionCalls.isEmpty(), "model should call a function tool"); - var spans = testHarness.awaitExportedSpans(); - assertEquals(1, spans.size()); - var span = spans.get(0); + // The LLM span plus one child tool span per function_call in the response output. + var spans = testHarness.awaitExportedSpans(1 + functionCalls.size()); + var span = llmSpan(spans); assertValidOpenAISpan(span, true); + assertEquals( + functionCalls.size(), + toolSpans(spans).size(), + "each function_call should produce a child tool span"); + assertTrue( + toolSpans(spans).stream() + .allMatch(t -> t.getParentSpanId().equals(span.getSpanId())), + "tool spans must be children of the LLM span"); // The Responses API serializes output as an "output" array; the streamed function_call // items must survive reconstruction into the span exactly as the client accumulated them. @@ -609,6 +617,26 @@ private static FunctionTool.Parameters weatherParameters( .build(); } + /** The single LLM span among exported spans (span_attributes type == "llm"). */ + @SneakyThrows + private static SpanData llmSpan(List spans) { + var llm = spans.stream().filter(s -> isSpanType(s, "llm")).toList(); + assertEquals(1, llm.size(), "expected exactly one LLM span"); + return llm.get(0); + } + + /** Child tool spans (span_attributes type == "tool") among exported spans. */ + private static List toolSpans(List spans) { + return spans.stream().filter(s -> isSpanType(s, "tool")).toList(); + } + + @SneakyThrows + private static boolean isSpanType(SpanData span, String type) { + String attr = + span.getAttributes().get(AttributeKey.stringKey("braintrust.span_attributes")); + return attr != null && type.equals(JSON_MAPPER.readTree(attr).path("type").asText()); + } + @SneakyThrows private JsonNode spanOutput(SpanData span) { String outputJson = diff --git a/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAIWebSearchTest.java b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAIWebSearchTest.java new file mode 100644 index 00000000..ae5111cf --- /dev/null +++ b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAIWebSearchTest.java @@ -0,0 +1,143 @@ +package dev.braintrust.instrumentation.openai.v2_15_0; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.client.OpenAIClient; +import com.openai.client.okhttp.OpenAIOkHttpClient; +import com.openai.core.http.StreamResponse; +import com.openai.helpers.ResponseAccumulator; +import com.openai.models.ChatModel; +import com.openai.models.responses.EasyInputMessage; +import com.openai.models.responses.Response; +import com.openai.models.responses.ResponseCreateParams; +import com.openai.models.responses.ResponseInputItem; +import com.openai.models.responses.ResponseStreamEvent; +import com.openai.models.responses.WebSearchTool; +import dev.braintrust.TestHarness; +import dev.braintrust.instrumentation.Instrumenter; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.trace.data.SpanData; +import java.util.List; +import lombok.SneakyThrows; +import net.bytebuddy.agent.ByteBuddyAgent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Verifies that built-in web-search tool calls in the OpenAI Responses API are captured as child + * {@code type:"tool"} spans parented to the LLM span, giving web search its own cost/latency + * visibility on the trace timeline. + */ +public class BraintrustOpenAIWebSearchTest { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + private static final AttributeKey SPAN_ATTRIBUTES = + AttributeKey.stringKey("braintrust.span_attributes"); + private static final AttributeKey METADATA = + AttributeKey.stringKey("braintrust.metadata"); + + @BeforeAll + public static void beforeAll() { + var instrumentation = ByteBuddyAgent.install(); + Instrumenter.install(instrumentation, BraintrustOpenAIWebSearchTest.class.getClassLoader()); + } + + private TestHarness testHarness; + + @BeforeEach + void beforeEach() { + testHarness = TestHarness.setup(); + } + + private static ResponseCreateParams webSearchRequest() { + return ResponseCreateParams.builder() + .model(ChatModel.GPT_4O) + .inputOfResponse( + List.of( + ResponseInputItem.ofEasyInputMessage( + EasyInputMessage.builder() + .role(EasyInputMessage.Role.USER) + .content( + "What is one recent headline about" + + " artificial intelligence? Use web" + + " search.") + .build()))) + .addTool( + WebSearchTool.builder().type(WebSearchTool.Type.WEB_SEARCH_PREVIEW).build()) + .build(); + } + + @Test + @SneakyThrows + void testResponsesWebSearch() { + OpenAIClient client = + OpenAIOkHttpClient.builder() + .baseUrl(testHarness.openAiBaseUrl()) + .apiKey(testHarness.openAiApiKey()) + .build(); + + Response response = client.responses().create(webSearchRequest()); + assertNotNull(response); + + var spans = testHarness.awaitExportedSpans(2); + assertWebSearchToolSpans(spans); + } + + @Test + @SneakyThrows + void testResponsesWebSearchStreaming() { + OpenAIClient client = + OpenAIOkHttpClient.builder() + .baseUrl(testHarness.openAiBaseUrl()) + .apiKey(testHarness.openAiApiKey()) + .build(); + + var accumulator = ResponseAccumulator.create(); + try (StreamResponse stream = + client.responses().createStreaming(webSearchRequest())) { + stream.stream().forEach(accumulator::accumulate); + } + assertFalse(accumulator.response().output().isEmpty(), "should generate a response"); + + var spans = testHarness.awaitExportedSpans(2); + assertWebSearchToolSpans(spans); + } + + @SneakyThrows + private static void assertWebSearchToolSpans(List spans) { + // Exactly one LLM span (the Responses request), plus one or more tool spans. + var llmSpans = spans.stream().filter(s -> isType(s, "llm")).toList(); + assertEquals(1, llmSpans.size(), "expected a single LLM span"); + var llm = llmSpans.get(0); + + var webSearchSpans = + spans.stream() + .filter(s -> isType(s, "tool")) + .filter(s -> "web_search_call".equals(s.getName())) + .toList(); + assertFalse( + webSearchSpans.isEmpty(), + "expected at least one web_search_call tool span, got spans: " + + spans.stream().map(SpanData::getName).toList()); + + for (var ws : webSearchSpans) { + assertEquals( + llm.getSpanId(), + ws.getParentSpanId(), + "web_search_call tool span must be a child of the LLM span"); + JsonNode metadata = JSON_MAPPER.readTree(ws.getAttributes().get(METADATA)); + assertEquals("web_search_call", metadata.path("tool_type").asText()); + } + } + + @SneakyThrows + private static boolean isType(SpanData span, String type) { + String attr = span.getAttributes().get(SPAN_ATTRIBUTES); + if (attr == null) { + return false; + } + return type.equals(JSON_MAPPER.readTree(attr).path("type").asText()); + } +} diff --git a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/ResponseToolSpans.java b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/ResponseToolSpans.java new file mode 100644 index 00000000..c25c7e7f --- /dev/null +++ b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/ResponseToolSpans.java @@ -0,0 +1,236 @@ +package dev.braintrust.instrumentation; + +import static dev.braintrust.json.BraintrustJsonMapper.toJson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import dev.braintrust.json.BraintrustJsonMapper; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; + +/** + * Emits child {@code type:"tool"} spans for built-in / server-side tool calls (web search, file + * search, code interpreter, etc.) that providers report inline with an LLM response. These make + * each tool call — most importantly web search, which carries its own cost and latency — visible on + * the trace timeline as a child of the LLM span. + * + *

The LLM-span tagging in {@link InstrumentationSemConv} is static and only has a {@link Span}; + * creating children requires a {@link Tracer} and the parent {@link Context}, both of which live in + * each provider's {@code TracingHttpClient}. So this logic lives here and is invoked from the + * client after the LLM span is tagged. + */ +@Slf4j +public final class ResponseToolSpans { + private ResponseToolSpans() {} + + private static final String TYPE_TOOL_JSON = "{\"type\":\"tool\"}"; + + // ------------------------------------------------------------------------- + // OpenAI Responses API + // ------------------------------------------------------------------------- + + /** + * Built-in tool item types found in the OpenAI Responses {@code output} array, mapped to the + * item fields that make up the tool span's input. Only these types produce spans. Mirrors the + * Python SDK's {@code _RESPONSE_TOOL_ITEM_INPUT_KEYS}. + */ + private static final Map> OPENAI_TOOL_ITEM_INPUT_KEYS = + Map.of( + "function_call", List.of("arguments"), + "web_search_call", List.of("action"), + "file_search_call", List.of("queries"), + "code_interpreter_call", List.of("code", "container_id"), + "computer_call", List.of("action"), + "image_generation_call", List.of(), + "mcp_call", List.of("arguments")); + + /** + * Parse an OpenAI Responses API response body and emit a child tool span for each built-in tool + * call in its {@code output} array. Chat Completions responses (no {@code output} array) yield + * no spans, so this is safe to call for any OpenAI response. + */ + public static void emitOpenAIResponseToolSpans( + Tracer tracer, Context parentContext, String responseBody) { + try { + JsonNode root = BraintrustJsonMapper.get().readTree(responseBody); + JsonNode output = root.get("output"); + if (output == null || !output.isArray()) { + return; + } + for (JsonNode item : output) { + if (!item.isObject()) { + continue; + } + String type = item.path("type").asText(null); + if (type == null || !OPENAI_TOOL_ITEM_INPUT_KEYS.containsKey(type)) { + continue; + } + emitOpenAIToolSpan(tracer, parentContext, item, type); + } + } catch (Exception e) { + log.debug("Could not emit OpenAI response tool spans", e); + } + } + + private static void emitOpenAIToolSpan( + Tracer tracer, Context parentContext, JsonNode item, String type) { + Span span = + tracer.spanBuilder(openAIToolSpanName(item, type)) + .setParent(parentContext) + .startSpan(); + try { + span.setAttribute("braintrust.span_attributes", TYPE_TOOL_JSON); + + JsonNode input = openAIToolSpanInput(item, type); + if (input != null && !input.isNull()) { + span.setAttribute("braintrust.input_json", toJson(input)); + } + String metadata = openAIToolSpanMetadata(item, type); + if (metadata != null) { + span.setAttribute("braintrust.metadata", metadata); + } + + // When the tool call errored, record the error and skip output (matches Python). + JsonNode error = openAIToolSpanError(item); + if (error != null) { + span.setStatus( + StatusCode.ERROR, error.isValueNode() ? error.asText() : error.toString()); + return; + } + JsonNode output = openAIToolSpanOutput(item, type); + if (output != null) { + span.setAttribute("braintrust.output_json", toJson(output)); + } + } catch (Exception e) { + log.debug("Could not tag OpenAI tool span", e); + } finally { + span.end(); + } + } + + private static String openAIToolSpanName(JsonNode item, String type) { + String serverLabel = nonEmptyText(item, "server_label"); + String name = nonEmptyText(item, "name"); + if (serverLabel != null && name != null) { + return serverLabel + "." + name; + } + if (name != null) { + return name; + } + return type; + } + + private static JsonNode openAIToolSpanInput(JsonNode item, String type) { + List inputKeys = OPENAI_TOOL_ITEM_INPUT_KEYS.get(type); + if (inputKeys.isEmpty()) { + return null; + } + ObjectNode inputData = BraintrustJsonMapper.get().createObjectNode(); + for (String key : inputKeys) { + JsonNode value = item.get(key); + if (value != null && !value.isNull()) { + inputData.set(key, maybeParseJsonString(value)); + } + } + if (inputData.isEmpty()) { + return null; + } + // Function/MCP calls carry a single `arguments` blob — unwrap it to the bare value. + if (inputKeys.size() == 1 && "arguments".equals(inputKeys.get(0))) { + return inputData.get("arguments"); + } + return inputData; + } + + private static JsonNode openAIToolSpanOutput(JsonNode item, String type) { + if ("function_call".equals(type)) { + return null; + } + Set excluded = + new HashSet<>(Set.of("id", "type", "name", "call_id", "server_label", "error")); + excluded.addAll(OPENAI_TOOL_ITEM_INPUT_KEYS.get(type)); + + ObjectNode output = BraintrustJsonMapper.get().createObjectNode(); + var fields = item.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + String key = entry.getKey(); + JsonNode value = entry.getValue(); + if (excluded.contains(key) || value == null || value.isNull()) { + continue; + } + if ("output".equals(key) || "error".equals(key)) { + output.set(key, maybeParseJsonString(value)); + } else { + output.set(key, value); + } + } + return output.isEmpty() ? null : output; + } + + private static JsonNode openAIToolSpanError(JsonNode item) { + JsonNode error = item.get("error"); + if (error == null || error.isNull()) { + return null; + } + return maybeParseJsonString(error); + } + + private static String openAIToolSpanMetadata(JsonNode item, String type) { + ObjectNode md = BraintrustJsonMapper.get().createObjectNode(); + md.put("tool_type", type); + putIfPresent(md, "tool_id", item.get("id")); + putIfPresent(md, "call_id", item.get("call_id")); + putIfPresent(md, "status", item.get("status")); + putIfPresent(md, "server_label", item.get("server_label")); + return md.isEmpty() ? null : toJson(md); + } + + // ------------------------------------------------------------------------- + // Shared helpers + // ------------------------------------------------------------------------- + + /** + * If {@code value} is a string that looks like JSON (starts with {@code [} or {@code {}), parse + * and return it as a tree; otherwise return it unchanged. Mirrors the Python SDK's {@code + * _maybe_parse_json_string}. + */ + private static JsonNode maybeParseJsonString(JsonNode value) { + if (value == null || !value.isTextual()) { + return value; + } + String stripped = value.asText().strip(); + if (stripped.isEmpty() || (stripped.charAt(0) != '[' && stripped.charAt(0) != '{')) { + return value; + } + try { + return BraintrustJsonMapper.get().readTree(stripped); + } catch (Exception e) { + return value; + } + } + + private static void putIfPresent(ObjectNode target, String key, JsonNode value) { + if (value != null && !value.isNull()) { + target.set(key, value); + } + } + + private static String nonEmptyText(JsonNode item, String field) { + JsonNode node = item.get(field); + if (node != null && node.isValueNode()) { + String text = node.asText(); + if (!text.isEmpty()) { + return text; + } + } + return null; + } +} diff --git a/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/ResponseToolSpansTest.java b/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/ResponseToolSpansTest.java new file mode 100644 index 00000000..9ed0428a --- /dev/null +++ b/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/ResponseToolSpansTest.java @@ -0,0 +1,224 @@ +package dev.braintrust.instrumentation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import dev.braintrust.json.BraintrustJsonMapper; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class ResponseToolSpansTest { + + private static final AttributeKey SPAN_ATTRIBUTES = + AttributeKey.stringKey("braintrust.span_attributes"); + private static final AttributeKey INPUT_JSON = + AttributeKey.stringKey("braintrust.input_json"); + private static final AttributeKey OUTPUT_JSON = + AttributeKey.stringKey("braintrust.output_json"); + private static final AttributeKey METADATA = + AttributeKey.stringKey("braintrust.metadata"); + + private InMemorySpanExporter exporter; + private SdkTracerProvider tracerProvider; + private Tracer tracer; + + @BeforeEach + void setUp() { + exporter = InMemorySpanExporter.create(); + tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build(); + tracer = tracerProvider.get("test"); + } + + @AfterEach + void tearDown() { + tracerProvider.close(); + } + + /** Runs the emitter with an LLM parent span and returns only the emitted tool spans. */ + private List emitOpenAI(String responseBody) { + Span parent = tracer.spanBuilder("llm").startSpan(); + try { + ResponseToolSpans.emitOpenAIResponseToolSpans( + tracer, Context.current().with(parent), responseBody); + } finally { + parent.end(); + } + return exporter.getFinishedSpanItems().stream() + .filter(s -> !s.getName().equals("llm")) + .collect(Collectors.toList()); + } + + private static SpanData byName(List spans, String name) { + return spans.stream().filter(s -> s.getName().equals(name)).findFirst().orElseThrow(); + } + + private static JsonNode json(String s) { + return BraintrustJsonMapper.fromJson(s, JsonNode.class); + } + + @Test + void emitsWebSearchCallToolSpanParentedToLlm() { + String body = + """ + { + "id": "resp_1", + "output": [ + { + "id": "ws_1", + "type": "web_search_call", + "status": "completed", + "action": {"type": "search", "query": "braintrust observability"} + }, + { + "id": "msg_1", + "type": "message", + "content": [{"type": "output_text", "text": "here is the answer"}] + } + ], + "usage": {"input_tokens": 10, "output_tokens": 20} + } + """; + + List tools = emitOpenAI(body); + // Only the web_search_call becomes a tool span; the message item is ignored. + assertEquals(1, tools.size()); + + SpanData ws = byName(tools, "web_search_call"); + // Parented to the LLM span. + SpanData llm = byName(exporter.getFinishedSpanItems(), "llm"); + assertEquals(llm.getSpanId(), ws.getParentSpanId()); + + assertEquals("{\"type\":\"tool\"}", ws.getAttributes().get(SPAN_ATTRIBUTES)); + + // action is the sole input key -> object with the action content. + JsonNode input = json(ws.getAttributes().get(INPUT_JSON)); + assertEquals("search", input.path("action").path("type").asText()); + assertEquals("braintrust observability", input.path("action").path("query").asText()); + + // metadata carries tool_type / tool_id / status. + JsonNode metadata = json(ws.getAttributes().get(METADATA)); + assertEquals("web_search_call", metadata.path("tool_type").asText()); + assertEquals("ws_1", metadata.path("tool_id").asText()); + assertEquals("completed", metadata.path("status").asText()); + + // action is an input key and id/type/error are excluded; the remaining `status` field + // flows into output (matching the Python SDK). + JsonNode output = json(ws.getAttributes().get(OUTPUT_JSON)); + assertEquals("completed", output.path("status").asText()); + } + + @Test + void functionCallUnwrapsArgumentsAndHasNoOutput() { + String body = + """ + { + "output": [ + { + "id": "fc_1", + "type": "function_call", + "call_id": "call_1", + "name": "get_weather", + "arguments": "{\\"city\\": \\"sf\\"}" + } + ] + } + """; + + List tools = emitOpenAI(body); + assertEquals(1, tools.size()); + SpanData fc = byName(tools, "get_weather"); + + // arguments unwrapped to the bare (parsed) value. + JsonNode input = json(fc.getAttributes().get(INPUT_JSON)); + assertEquals("sf", input.path("city").asText()); + + // function_call never produces output. + assertNull(fc.getAttributes().get(OUTPUT_JSON)); + + JsonNode metadata = json(fc.getAttributes().get(METADATA)); + assertEquals("function_call", metadata.path("tool_type").asText()); + assertEquals("call_1", metadata.path("call_id").asText()); + } + + @Test + void errorItemSetsErrorStatusAndSkipsOutput() { + String body = + """ + { + "output": [ + { + "id": "ws_2", + "type": "web_search_call", + "status": "failed", + "action": {"query": "x"}, + "error": "rate_limited" + } + ] + } + """; + + List tools = emitOpenAI(body); + assertEquals(1, tools.size()); + SpanData ws = tools.get(0); + assertEquals(StatusCode.ERROR, ws.getStatus().getStatusCode()); + assertNull(ws.getAttributes().get(OUTPUT_JSON)); + } + + @Test + void mcpCallUsesServerLabelDottedName() { + String body = + """ + { + "output": [ + { + "id": "mcp_1", + "type": "mcp_call", + "server_label": "deepwiki", + "name": "ask_question", + "arguments": "{\\"q\\": \\"hi\\"}", + "output": "an answer" + } + ] + } + """; + + List tools = emitOpenAI(body); + assertEquals(1, tools.size()); + SpanData mcp = byName(tools, "deepwiki.ask_question"); + // output is present (mcp_call is not function_call) and JSON-parsed where applicable. + assertEquals( + "an answer", json(mcp.getAttributes().get(OUTPUT_JSON)).path("output").asText()); + } + + @Test + void nonToolResponseEmitsNothing() { + // A Chat Completions body (no `output` array) yields no tool spans. + String body = + """ + {"choices": [{"message": {"content": "hi"}}], "usage": {"total_tokens": 5}} + """; + assertTrue(emitOpenAI(body).isEmpty()); + } + + @Test + void malformedBodyIsIgnored() { + assertTrue(emitOpenAI("not json").isEmpty()); + } +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-0c37df81d2cf.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-0c37df81d2cf.json new file mode 100644 index 00000000..efd3c7cb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-0c37df81d2cf.json @@ -0,0 +1,127 @@ +{ + "id": "resp_050cff5ce90128c7006a7fc279ddc487d08d20eb4e7cbf2ae6", + "object": "response", + "created_at": 1786757753, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1786757760, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-2024-08-06", + "moderation": null, + "output": [ + { + "id": "ws_050cff5ce90128c7006a7fc27d7c8c87d0919a4bacb3b2679e", + "type": "web_search_call", + "status": "completed", + "action": { + "type": "search", + "queries": [ + "recent news about artificial intelligence" + ], + "query": "recent news about artificial intelligence" + } + }, + { + "id": "msg_050cff5ce90128c7006a7fc27f26a887d0ae49e55d9edeb9f2", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [ + { + "type": "url_citation", + "end_index": 493, + "start_index": 392, + "title": "2025 in artificial intelligence", + "url": "https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai" + } + ], + "logprobs": [], + "text": "As of August 2026, a notable development in artificial intelligence is the release of OpenAI's GPT-5 on August 8, 2025. According to OpenAI CEO Sam Altman, GPT-5 is designed to provide PhD-level expertise. However, it has faced challenges with basic tasks such as spelling and geography, and it cannot continuously learn. Additionally, GPT-5 scores lower than Grok-4 on Humanity's Last Exam. ([en.wikipedia.org](https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai)) " + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 1 + } + }, + "tools": [ + { + "type": "web_search_preview", + "search_content_types": [ + "text" + ], + "search_context_size": "medium", + "user_location": { + "type": "approximate", + "city": null, + "country": "US", + "region": null, + "timezone": null + } + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 314, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 134, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 448 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-31d9a6eee624.txt b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-31d9a6eee624.txt new file mode 100644 index 00000000..84aff527 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-31d9a6eee624.txt @@ -0,0 +1,315 @@ +event: response.created +data: {"type":"response.created","response":{"id":"resp_0dddc08e04ed0eef006a7fc272fa6c819a97e0ce827ab581a3","object":"response","created_at":1786757747,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"web_search_preview","search_content_types":["text"],"search_context_size":"medium","user_location":{"type":"approximate","city":null,"country":"US","region":null,"timezone":null}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + +event: response.in_progress +data: {"type":"response.in_progress","response":{"id":"resp_0dddc08e04ed0eef006a7fc272fa6c819a97e0ce827ab581a3","object":"response","created_at":1786757747,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"web_search_preview","search_content_types":["text"],"search_context_size":"medium","user_location":{"type":"approximate","city":null,"country":"US","region":null,"timezone":null}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"ws_0dddc08e04ed0eef006a7fc276c298819a835576df99f28ffb","type":"web_search_call","status":"in_progress","action":{"type":"search"}},"output_index":0,"sequence_number":2} + +event: response.web_search_call.in_progress +data: {"type":"response.web_search_call.in_progress","item_id":"ws_0dddc08e04ed0eef006a7fc276c298819a835576df99f28ffb","output_index":0,"sequence_number":3} + +event: response.web_search_call.searching +data: {"type":"response.web_search_call.searching","item_id":"ws_0dddc08e04ed0eef006a7fc276c298819a835576df99f28ffb","output_index":0,"sequence_number":4} + +event: response.web_search_call.completed +data: {"type":"response.web_search_call.completed","item_id":"ws_0dddc08e04ed0eef006a7fc276c298819a835576df99f28ffb","output_index":0,"sequence_number":5} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"ws_0dddc08e04ed0eef006a7fc276c298819a835576df99f28ffb","type":"web_search_call","status":"completed","action":{"type":"search","queries":["recent headline about artificial intelligence"],"query":"recent headline about artificial intelligence"}},"output_index":0,"sequence_number":6} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":1,"sequence_number":7} + +event: response.content_part.added +data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":8} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"As","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"p4iQfM4F2I53x1","output_index":1,"sequence_number":9} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" of","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"GyPNTZe5a7NDH","output_index":1,"sequence_number":10} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" August","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"XrzRFgSSO","output_index":1,"sequence_number":11} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" ","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"kIqSanYCBDRc86O","output_index":1,"sequence_number":12} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"202","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"Ay77x92qZGltt","output_index":1,"sequence_number":13} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"6","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"8pvLFQgfF0H9Xld","output_index":1,"sequence_number":14} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":",","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"0YOyEccpeB120pF","output_index":1,"sequence_number":15} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" a","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"2HcykqUAcpiNuh","output_index":1,"sequence_number":16} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" notable","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"utTGr5al","output_index":1,"sequence_number":17} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" development","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"qb3V","output_index":1,"sequence_number":18} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" in","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"C1I5IgBCS9t5V","output_index":1,"sequence_number":19} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" artificial","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"aucNp","output_index":1,"sequence_number":20} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" intelligence","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"ZBe","output_index":1,"sequence_number":21} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" is","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"kYcbIbwDnwN9x","output_index":1,"sequence_number":22} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" the","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"MLFzNkDDOSdv","output_index":1,"sequence_number":23} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" release","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"78OGYLoC","output_index":1,"sequence_number":24} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" of","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"OgYLbuaqsHcF9","output_index":1,"sequence_number":25} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" Open","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"rilvox8JPb8","output_index":1,"sequence_number":26} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"AI","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"oPxb5svp4sAeyw","output_index":1,"sequence_number":27} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"'s","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"ZnF5O64CEgwAiq","output_index":1,"sequence_number":28} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" GPT","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"8MRKcz8iCYR6","output_index":1,"sequence_number":29} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"-","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"MXIpsXsT4GIRXQJ","output_index":1,"sequence_number":30} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"5","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"ak22ZoyUw4v3sFk","output_index":1,"sequence_number":31} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" on","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"rEyPABRrS0A4z","output_index":1,"sequence_number":32} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" August","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"mUezIJydq","output_index":1,"sequence_number":33} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" ","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"PTEWWbc7Ngnq8N2","output_index":1,"sequence_number":34} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"8","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"bc5nrccDmFZ8Wcw","output_index":1,"sequence_number":35} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":",","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"gtdq2ureC8QOvBu","output_index":1,"sequence_number":36} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" ","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"OS5yEdaf17bmJpS","output_index":1,"sequence_number":37} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"202","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"naDma9Mcp85aA","output_index":1,"sequence_number":38} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"5","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"LXbhAMN4O1myfQV","output_index":1,"sequence_number":39} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"N0nfKYosZKT2mZ0","output_index":1,"sequence_number":40} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" ","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"zcO9rPzUeRuFLwG","output_index":1,"sequence_number":41} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"According","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"zkPnirZ","output_index":1,"sequence_number":42} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" to","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"ctaPSl5v0AOgW","output_index":1,"sequence_number":43} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" Open","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"WVWyk93tP0y","output_index":1,"sequence_number":44} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"AI","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"bGJMHk3w8ne2fp","output_index":1,"sequence_number":45} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" CEO","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"kNRuFNc0NTzo","output_index":1,"sequence_number":46} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" Sam","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"vFtK7W3csUkK","output_index":1,"sequence_number":47} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" Alt","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"lpn7nr7nstC9","output_index":1,"sequence_number":48} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"man","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"FruMXyLIaSeys","output_index":1,"sequence_number":49} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":",","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"fPIakSosAHxeCz5","output_index":1,"sequence_number":50} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" GPT","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"keAIx3PgsfZE","output_index":1,"sequence_number":51} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"-","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"J6CTvqT4orCKpca","output_index":1,"sequence_number":52} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"5","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"ekNf9KB0d4mGeju","output_index":1,"sequence_number":53} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" is","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"AWq3Ybj63KKK1","output_index":1,"sequence_number":54} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" designed","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"6LMw6f3","output_index":1,"sequence_number":55} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" to","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"tw9KPbMWmOG0m","output_index":1,"sequence_number":56} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" provide","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"hmO3iq4M","output_index":1,"sequence_number":57} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" Ph","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"puGg2CqytCgzE","output_index":1,"sequence_number":58} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"D","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"azbfYCVSjwgmPpk","output_index":1,"sequence_number":59} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"-level","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"BBPquq7N8W","output_index":1,"sequence_number":60} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" expertise","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"4Dm0QN","output_index":1,"sequence_number":61} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"cgRkWV85SovW3AI","output_index":1,"sequence_number":62} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" ","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"B0pppZVMmoKT8LK","output_index":1,"sequence_number":63} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"However","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"7GJm1DJlo","output_index":1,"sequence_number":64} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":",","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"IiaNduaCAgHdPp9","output_index":1,"sequence_number":65} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" it","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"yWhjIFhnjVOkf","output_index":1,"sequence_number":66} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" has","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"QgFp1oyP81W1","output_index":1,"sequence_number":67} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" faced","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"7EZ47PxKs5","output_index":1,"sequence_number":68} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" challenges","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"8Xllj","output_index":1,"sequence_number":69} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" with","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"CrqQNaFCNfj","output_index":1,"sequence_number":70} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" basic","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"LbOMS5CH1a","output_index":1,"sequence_number":71} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" tasks","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"eB7svHO6yR","output_index":1,"sequence_number":72} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" such","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"NE3WbfpeThM","output_index":1,"sequence_number":73} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" as","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"E48VJwCEHM38W","output_index":1,"sequence_number":74} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" spelling","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"nhXx9ij","output_index":1,"sequence_number":75} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" and","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"FinC3Ho9uh1A","output_index":1,"sequence_number":76} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" geography","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"8MZR2l","output_index":1,"sequence_number":77} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":",","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"0V4WYtRZjhh3q1A","output_index":1,"sequence_number":78} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" cannot","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"nn8b1DBij","output_index":1,"sequence_number":79} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" continuously","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"OyE","output_index":1,"sequence_number":80} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" learn","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"LE5Ncno4vq","output_index":1,"sequence_number":81} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":",","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"BY9koCdsY5iql2d","output_index":1,"sequence_number":82} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" and","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"uSyW4yXmbgRc","output_index":1,"sequence_number":83} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" scores","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"Y9mSdUoUx","output_index":1,"sequence_number":84} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" lower","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"4ZP7WylMAy","output_index":1,"sequence_number":85} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" than","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"BjOfLgq8Fej","output_index":1,"sequence_number":86} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" Gro","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"pCUaWZohC16Y","output_index":1,"sequence_number":87} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"k","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"Xz6Ude073ndfoMo","output_index":1,"sequence_number":88} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"-","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"rZ4c0naVSCLtF8o","output_index":1,"sequence_number":89} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"4","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"Tn8ANS6WLtOjDlx","output_index":1,"sequence_number":90} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" on","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"GX4nftA8B8sFk","output_index":1,"sequence_number":91} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" Humanity","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"WCsVI0t","output_index":1,"sequence_number":92} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"'s","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"jL4nv4KLTUkgf2","output_index":1,"sequence_number":93} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" Last","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"4Upt4GZubsA","output_index":1,"sequence_number":94} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" Exam","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"QbqrzTrVfbx","output_index":1,"sequence_number":95} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"2fUWX8SyOBDFGKl","output_index":1,"sequence_number":96} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" ","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"fQ1KKp0qBGeu5np","output_index":1,"sequence_number":97} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"([en.wikipedia.org](https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai))","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"tn0wd1lHZiq","output_index":1,"sequence_number":98} + +event: response.output_text.annotation.added +data: {"type":"response.output_text.annotation.added","annotation":{"type":"url_citation","end_index":470,"start_index":369,"title":"2025 in artificial intelligence","url":"https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai"},"annotation_index":0,"content_index":0,"item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","output_index":1,"sequence_number":99} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" ","item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"obfuscation":"rccxILlSOtR6D4d","output_index":1,"sequence_number":100} + +event: response.output_text.done +data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","logprobs":[],"output_index":1,"sequence_number":101,"text":"As of August 2026, a notable development in artificial intelligence is the release of OpenAI's GPT-5 on August 8, 2025. According to OpenAI CEO Sam Altman, GPT-5 is designed to provide PhD-level expertise. However, it has faced challenges with basic tasks such as spelling and geography, cannot continuously learn, and scores lower than Grok-4 on Humanity's Last Exam. ([en.wikipedia.org](https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai)) "} + +event: response.content_part.done +data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","output_index":1,"part":{"type":"output_text","annotations":[{"type":"url_citation","end_index":470,"start_index":369,"title":"2025 in artificial intelligence","url":"https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai"}],"logprobs":[],"text":"As of August 2026, a notable development in artificial intelligence is the release of OpenAI's GPT-5 on August 8, 2025. According to OpenAI CEO Sam Altman, GPT-5 is designed to provide PhD-level expertise. However, it has faced challenges with basic tasks such as spelling and geography, cannot continuously learn, and scores lower than Grok-4 on Humanity's Last Exam. ([en.wikipedia.org](https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai)) "},"sequence_number":102} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","type":"message","status":"completed","content":[{"type":"output_text","annotations":[{"type":"url_citation","end_index":470,"start_index":369,"title":"2025 in artificial intelligence","url":"https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai"}],"logprobs":[],"text":"As of August 2026, a notable development in artificial intelligence is the release of OpenAI's GPT-5 on August 8, 2025. According to OpenAI CEO Sam Altman, GPT-5 is designed to provide PhD-level expertise. However, it has faced challenges with basic tasks such as spelling and geography, cannot continuously learn, and scores lower than Grok-4 on Humanity's Last Exam. ([en.wikipedia.org](https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai)) "}],"role":"assistant"},"output_index":1,"sequence_number":103} + +event: response.completed +data: {"type":"response.completed","response":{"id":"resp_0dddc08e04ed0eef006a7fc272fa6c819a97e0ce827ab581a3","object":"response","created_at":1786757747,"status":"completed","background":false,"completed_at":1786757753,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[{"id":"ws_0dddc08e04ed0eef006a7fc276c298819a835576df99f28ffb","type":"web_search_call","status":"completed","action":{"type":"search","queries":["recent headline about artificial intelligence"],"query":"recent headline about artificial intelligence"}},{"id":"msg_0dddc08e04ed0eef006a7fc277c2d0819aa2b460e1b35c087b","type":"message","status":"completed","content":[{"type":"output_text","annotations":[{"type":"url_citation","end_index":470,"start_index":369,"title":"2025 in artificial intelligence","url":"https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai"}],"logprobs":[],"text":"As of August 2026, a notable development in artificial intelligence is the release of OpenAI's GPT-5 on August 8, 2025. According to OpenAI CEO Sam Altman, GPT-5 is designed to provide PhD-level expertise. However, it has faced challenges with basic tasks such as spelling and geography, cannot continuously learn, and scores lower than Grok-4 on Humanity's Last Exam. ([en.wikipedia.org](https://en.wikipedia.org/wiki/2025_in_artificial_intelligence?utm_source=openai)) "}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":1}},"tools":[{"type":"web_search_preview","search_content_types":["text"],"search_context_size":"medium","user_location":{"type":"approximate","city":null,"country":"US","region":null,"timezone":null}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":314,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":129,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":443},"user":null,"metadata":{}},"sequence_number":104} + diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-0c37df81d2cf.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-0c37df81d2cf.json new file mode 100644 index 00000000..4cf01a06 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-0c37df81d2cf.json @@ -0,0 +1,48 @@ +{ + "id" : "281f816f-d189-3cf5-88f9-fd4fdc9fe66d", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":[{\"content\":\"What is one recent headline about artificial intelligence? Use web search.\",\"role\":\"user\"}],\"model\":\"gpt-4o\",\"tools\":[{\"type\":\"web_search_preview\"}]}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-0c37df81d2cf.json", + "headers" : { + "x-request-id" : "req_caffbff4fcfa4447b1f0f244b349c260", + "x-ratelimit-limit-tokens" : "30000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2b477186f387696-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "6ms", + "x-ratelimit-remaining-tokens" : "29999667", + "x-ratelimit-remaining-requests" : "9999", + "Date" : "Sat, 15 Aug 2026 01:36:00 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=fKB9N5Strq0k5Ze6sBfi4jGPWxkXhYwFN8qyUXdPsIM-1786757753.6642315-1.0.1.1-wyZZ9kuB_61Xx6odgjT2OxPYQpP8j3xce70cTL5rB304kOCWgffE6Fj.BtDgeYCAyjCkI7SOuZ4KDifin5OZNXNfsEcE5OgeU7w.lKUK9iMdxo4FO.1aWZFgahNgnkqF; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Sat, 15 Aug 2026 02:06:00 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "10000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "6801", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "281f816f-d189-3cf5-88f9-fd4fdc9fe66d", + "persistent" : true, + "insertionIndex" : 38 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-31d9a6eee624.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-31d9a6eee624.json new file mode 100644 index 00000000..5718205d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-31d9a6eee624.json @@ -0,0 +1,42 @@ +{ + "id" : "4ecd239c-fa2c-3a94-9038-bb602a460baa", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":[{\"content\":\"What is one recent headline about artificial intelligence? Use web search.\",\"role\":\"user\"}],\"model\":\"gpt-4o\",\"tools\":[{\"type\":\"web_search_preview\"}],\"stream\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-31d9a6eee624.txt", + "headers" : { + "x-request-id" : "req_2f1291ac8203453fb7476805c8eb5c01", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2b476dad9132537-SEA", + "X-Content-Type-Options" : "nosniff", + "Date" : "Sat, 15 Aug 2026 01:35:47 GMT", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=q6tyEpyrXqDMtWwq2JvbVBjmkxbDf7l2v1_lo5R3X98-1786757743.8158913-1.0.1.1-WFRbjAqwnEOAf4BovtQaPxjU47q0SMcLUlfBkd949mJ48TGnhQ4BTdgbCvT.aHYoHKwo4FBtK.gTmhGWs3Mtaca2Iq2TajrUg.ffrJwJR9cPWwHXRpjj9IVj1f7eEZIA; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Sat, 15 Aug 2026 02:05:47 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "198", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "text/event-stream; charset=utf-8" + } + }, + "uuid" : "4ecd239c-fa2c-3a94-9038-bb602a460baa", + "persistent" : true, + "insertionIndex" : 39 +} \ No newline at end of file