diff --git a/internal/execution/bifrost/converted.go b/internal/execution/bifrost/converted.go index 8cf32d543d..31a17b7dbf 100644 --- a/internal/execution/bifrost/converted.go +++ b/internal/execution/bifrost/converted.go @@ -127,10 +127,19 @@ func buildConvertedResponsesRequest(spec execution.AttemptSpec, provider schemas if request.Params == nil { request.Params = &schemas.ResponsesParameters{} } + stripResponsesControlParams(request.Params) if err := mergeConvertedParameterOverrides(spec, &request.Params.ExtraParams); err != nil { return nil, err } - stripResponsesControlParams(request.Params) + if spec.RouteMode == execution.RouteConverted && spec.ChannelID == string(channel.OpenRouter) { + // Only provider preferences use extra-parameter passthrough. The SDK + // still prepares and serializes the typed request before merging them. + preferences, exists := request.Params.ExtraParams["provider"] + request.Params.ExtraParams = nil + if exists { + request.Params.ExtraParams = map[string]any{"provider": preferences} + } + } return request, nil } diff --git a/internal/execution/bifrost/executor.go b/internal/execution/bifrost/executor.go index 8747db5f1f..bbae3c2896 100644 --- a/internal/execution/bifrost/executor.go +++ b/internal/execution/bifrost/executor.go @@ -1519,6 +1519,9 @@ func (r *Runtime) newSDKContext(parent context.Context, spec execution.AttemptSp spec.ClientProtocol == protocol.Anthropic { bifrostContext.SetValue(schemas.BifrostContextKeyPassthroughExtraParams, true) } + if spec.RouteMode == execution.RouteConverted && spec.ChannelID == string(channel.OpenRouter) { + bifrostContext.SetValue(schemas.BifrostContextKeyPassthroughExtraParams, true) + } if spec.Operation == execution.OperationProbe && openAICompatibleBase(r.providerKind(spec)) { bifrostContext.SetValue(schemas.BifrostContextKeyPassthroughExtraParams, true) diff --git a/internal/execution/bifrost/executor_test.go b/internal/execution/bifrost/executor_test.go index 32ed16e877..474348040c 100644 --- a/internal/execution/bifrost/executor_test.go +++ b/internal/execution/bifrost/executor_test.go @@ -855,8 +855,10 @@ func TestSanitizeNativeChatBodyPreservesExplicitStreamOptions(t *testing.T) { t.Parallel() body, err := sanitizeNativeChatBody( - []byte(`{"model":"client","messages":[],"stream_options":{"include_usage":false,"vendor_option":"keep"}}`), - "upstream", + execution.AttemptSpec{ + Body: []byte(`{"model":"client","messages":[],"stream_options":{"include_usage":false,"vendor_option":"keep"}}`), + UpstreamModel: "upstream", + }, true, ) if err != nil { diff --git a/internal/execution/bifrost/parameter_overrides.go b/internal/execution/bifrost/parameter_overrides.go index 583a0deeb8..47bdd25c16 100644 --- a/internal/execution/bifrost/parameter_overrides.go +++ b/internal/execution/bifrost/parameter_overrides.go @@ -10,6 +10,7 @@ import ( "github.com/maximhq/bifrost/core/providers/gemini" "github.com/maximhq/bifrost/core/providers/openai" + "gpt-load/internal/channel" "gpt-load/internal/execution" "gpt-load/internal/protocol" ) @@ -48,7 +49,8 @@ func sdkRequestFields(value reflect.Type) map[string]bool { } func convertedParameterOverrides(spec execution.AttemptSpec) (map[string]any, error) { - if spec.RouteMode != execution.RouteConverted || len(spec.ConfiguredParameters) == 0 { + allowProviderPreferences := spec.ChannelID == string(channel.OpenRouter) + if spec.RouteMode != execution.RouteConverted || len(spec.ConfiguredParameters) == 0 && !allowProviderPreferences { return nil, nil } known := convertedRequestFields[spec.ClientProtocol] @@ -60,9 +62,13 @@ func convertedParameterOverrides(spec execution.AttemptSpec) (map[string]any, er return nil, fmt.Errorf("invalid overridden request body") } // 复用原生请求的控制字段清理,扩展参数不能接管凭据或 SDK 路由。 - stripNativeControlFields(body) + stripNativeControlFields(body, allowProviderPreferences) + names := spec.ConfiguredParameters + if allowProviderPreferences { + names = append([]string{"provider"}, names...) + } extras := make(map[string]any) - for _, name := range spec.ConfiguredParameters { + for _, name := range names { lower := strings.ToLower(name) if known[lower] { continue diff --git a/internal/execution/bifrost/passthrough.go b/internal/execution/bifrost/passthrough.go index 7aaa41169f..4464564cfa 100644 --- a/internal/execution/bifrost/passthrough.go +++ b/internal/execution/bifrost/passthrough.go @@ -57,14 +57,14 @@ func nativeMessageProvider(providerKind channel.ProviderKind, spec execution.Att return "", false } -func sanitizeNativeChatBody(body []byte, upstreamModel string, stream ...bool) ([]byte, error) { - object, err := decodeNativeJSONObject(body) +func sanitizeNativeChatBody(spec execution.AttemptSpec, stream ...bool) ([]byte, error) { + object, err := decodeNativeJSONObject(spec.Body) if err != nil { return nil, err } - changed := stripNativeControlFields(object) - if !jsonStringEquals(object["model"], upstreamModel) { - if err := forceJSONModel(object, upstreamModel); err != nil { + changed := stripNativeControlFields(object, spec.ChannelID == string(channel.OpenRouter)) + if !jsonStringEquals(object["model"], spec.UpstreamModel) { + if err := forceJSONModel(object, spec.UpstreamModel); err != nil { return nil, err } changed = true @@ -77,7 +77,7 @@ func sanitizeNativeChatBody(body []byte, upstreamModel string, stream ...bool) ( } } if !changed { - return append([]byte(nil), body...), nil + return append([]byte(nil), spec.Body...), nil } return encodeNativeJSONObject(object) } @@ -88,7 +88,7 @@ func sanitizeNativeRequestBody(spec execution.AttemptSpec, stream bool) ([]byte, return nil, nil case (spec.ClientProtocol == protocol.OpenAICompletions || spec.ClientProtocol == protocol.Anthropic) && spec.Operation == execution.OperationChatCompletion: - body, err := sanitizeNativeChatBody(spec.Body, spec.UpstreamModel, stream) + body, err := sanitizeNativeChatBody(spec, stream) if err != nil { return nil, err } @@ -100,7 +100,7 @@ func sanitizeNativeRequestBody(spec execution.AttemptSpec, stream bool) ([]byte, } return body, nil case spec.ClientProtocol == protocol.Anthropic && spec.Operation == execution.OperationCountTokens: - return sanitizeNativeChatBody(spec.Body, spec.UpstreamModel) + return sanitizeNativeChatBody(spec) case spec.ClientProtocol == protocol.Gemini && spec.Operation == execution.OperationChatCompletion: object, err := decodeNativeJSONObject(spec.Body) if err != nil { @@ -108,7 +108,7 @@ func sanitizeNativeRequestBody(spec execution.AttemptSpec, stream bool) ([]byte, // validate, while valid JSON is scrubbed below. return append([]byte(nil), spec.Body...), nil } - changed := stripNativeControlFields(object) + changed := stripNativeControlFields(object, false) if !changed { return append([]byte(nil), spec.Body...), nil } @@ -118,7 +118,7 @@ func sanitizeNativeRequestBody(spec execution.AttemptSpec, stream bool) ([]byte, if err != nil { return append([]byte(nil), spec.Body...), nil } - if !stripNativeControlFields(object) { + if !stripNativeControlFields(object, false) { return append([]byte(nil), spec.Body...), nil } return encodeNativeJSONObject(object) @@ -128,7 +128,7 @@ func sanitizeNativeRequestBody(spec execution.AttemptSpec, stream bool) ([]byte, if err != nil { return nil, err } - changed := stripNativeControlFields(object) + changed := stripNativeControlFields(object, spec.ChannelID == string(channel.OpenRouter)) if !jsonStringEquals(object["model"], spec.UpstreamModel) { if err := forceJSONModel(object, spec.UpstreamModel); err != nil { return nil, err @@ -150,7 +150,7 @@ func sanitizeNativeRequestBody(spec execution.AttemptSpec, stream bool) ([]byte, if err != nil { return nil, err } - changed := stripNativeControlFields(object) + changed := stripNativeControlFields(object, false) if !jsonStringEquals(object["model"], spec.UpstreamModel) { if err := forceJSONModel(object, spec.UpstreamModel); err != nil { return nil, err @@ -173,7 +173,7 @@ func sanitizeNativeRequestBody(spec execution.AttemptSpec, stream bool) ([]byte, if err != nil { return nil, err } - changed := stripNativeControlFields(object) + changed := stripNativeControlFields(object, false) if _, hasModel := object["model"]; hasModel && strings.TrimSpace(spec.UpstreamModel) != "" { if !jsonStringEquals(object["model"], spec.UpstreamModel) { if err := forceJSONModel(object, spec.UpstreamModel); err != nil { @@ -194,7 +194,7 @@ func sanitizeNativeRequestBody(spec execution.AttemptSpec, stream bool) ([]byte, if err != nil { return nil, err } - stripNativeControlFields(object) + stripNativeControlFields(object, false) return encodeNativeJSONObject(object) } } @@ -252,9 +252,14 @@ func decodeNativeJSONObject(body []byte) (map[string]json.RawMessage, error) { return object, nil } -func stripNativeControlFields(object map[string]json.RawMessage) bool { +func stripNativeControlFields(object map[string]json.RawMessage, allowProviderPreferences bool) bool { changed := false - for key := range object { + for key, value := range object { + // OpenRouter's native provider object selects its upstream providers. + // Scalar gateway controls and case variants remain filtered. + if allowProviderPreferences && key == "provider" && bytes.HasPrefix(bytes.TrimSpace(value), []byte("{")) { + continue + } normalized := strings.ReplaceAll(strings.ToLower(key), "-", "_") switch normalized { case "provider", "fallback", "fallbacks", "authorization", "proxy_authorization", diff --git a/internal/gateway/openrouter_converted_provider_test.go b/internal/gateway/openrouter_converted_provider_test.go new file mode 100644 index 0000000000..4207c8f238 --- /dev/null +++ b/internal/gateway/openrouter_converted_provider_test.go @@ -0,0 +1,249 @@ +package gateway + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync/atomic" + "testing" + + "gpt-load/internal/channel" + "gpt-load/internal/dialect" + "gpt-load/internal/platform/config" + "gpt-load/internal/protocol" + "gpt-load/internal/state" +) + +func TestGatewayConvertedOpenRouterProviderPreferences(t *testing.T) { + preferences := map[string]any{"only": []any{"modal/fp4"}, "allow_fallbacks": false} + for _, test := range []struct { + name string + channel channel.ID + provider any + preserve bool + }{ + {"OpenRouter preferences", channel.OpenRouter, preferences, true}, + {"OpenRouter empty preferences", channel.OpenRouter, map[string]any{}, true}, + {"OpenRouter scalar control", channel.OpenRouter, "anthropic", false}, + {"OpenRouter null control", channel.OpenRouter, nil, false}, + {"OpenRouter array control", channel.OpenRouter, []any{"modal/fp4"}, false}, + {"OpenAI control", channel.OpenAI, preferences, false}, + } { + for _, client := range []protocol.Protocol{protocol.Anthropic, protocol.Gemini} { + for _, configured := range []bool{false, true} { + for _, stream := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/configured=%t/stream=%t", test.name, client, configured, stream), func(t *testing.T) { + var calls atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.Method != http.MethodPost || r.URL.Path != "/v1/responses" || r.Header.Get("Authorization") != "Bearer sk-upstream" { + t.Errorf("unexpected upstream route or credential: %s %s", r.Method, r.URL.Path) + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + return + } + if body["model"] != "anthropic/claude-sonnet-4.6" || (body["stream"] == true) != stream { + t.Errorf("model or stream changed: %#v", body) + } + if client == protocol.Anthropic { + if body["max_output_tokens"] != float64(16) { + t.Errorf("converted output limit = %#v", body["max_output_tokens"]) + } + if test.channel == channel.OpenRouter { + input := body["input"].([]any) + blocks := input[0].(map[string]any)["content"].([]any) + block := blocks[0].(map[string]any) + if block["text"] != "shared prefix" || !reflect.DeepEqual(block["prompt_cache_breakpoint"], map[string]any{"mode": "explicit"}) { + t.Errorf("cache marker not preserved: %#v", block) + } + } + } + if test.preserve { + if !reflect.DeepEqual(body["provider"], test.provider) { + t.Errorf("provider preferences = %#v, want %#v", body["provider"], test.provider) + } + } else if _, exists := body["provider"]; exists { + t.Errorf("provider control reached upstream: %#v", body["provider"]) + } + for _, key := range []string{"Provider", "fallback", "fallbacks", "api_key", "Authorization", "proxy-authorization", "x-api-key", "x-goog-api-key", "client_extra", "custom_option", "top_k"} { + if _, exists := body[key]; exists { + t.Errorf("control %q reached upstream", key) + } + } + response := `{"id":"resp_1","object":"response","model":"anthropic/claude-sonnet-4.6","status":"completed","output":[{"id":"msg_1","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"ok","annotations":[]}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}` + w.Header().Set("Content-Type", "application/json") + if stream { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":"+response+"}\n\n") + } else { + _, _ = io.WriteString(w, response) + } + })) + defer upstream.Close() + controls := map[string]any{"provider": test.provider, "Provider": preferences, "fallback": "other", "fallbacks": []any{"other"}, "api_key": "client-key", "Authorization": "client-key", "proxy-authorization": "client-key", "x-api-key": "client-key", "x-goog-api-key": "client-key", "custom_option": true, "top_k": 7} + body := map[string]any{"model": "public-model", "stream": stream, "client_extra": "drop"} + path := "/v1/messages" + if client == protocol.Anthropic { + body["max_tokens"] = 16 + body["system"] = []any{map[string]any{"type": "text", "text": "shared prefix", "cache_control": map[string]any{"type": "ephemeral"}}} + body["messages"] = []any{map[string]any{"role": "user", "content": "hello"}} + } else { + path = "/v1beta/models/public-model:generateContent" + if stream { + path = "/v1beta/models/public-model:streamGenerateContent?alt=sse" + } + body["contents"] = []any{map[string]any{"role": "user", "parts": []any{map[string]any{"text": "hello"}}}} + } + var settings config.Settings + if configured { + body["provider"] = "client-control" + settings = config.Settings{state.SettingParameterOverrides: []any{map[string]any{"set": controls}}} + } else { + for key, value := range controls { + body[key] = value + } + } + params, _ := json.Marshal(map[string]string{"base_url": upstream.URL}) + engine, _ := newDialectGatewayEngine(t, client, "public-model", dialect.NewSet(dialect.NewAnthropic(), dialect.NewGemini()), dialectGatewayGroup{ + id: 1, name: "test", channelID: test.channel, params: params, apiKeys: []string{"sk-upstream"}, + models: []state.ModelConfig{{ID: "anthropic/claude-sonnet-4.6", Alias: "public-model"}}, settings: settings, + }) + encoded, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, path, strings.NewReader(string(encoded))) + request.Header.Set("Authorization", "Bearer gl-client") + recorder := httptest.NewRecorder() + engine.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK || calls.Load() != 1 { + t.Fatalf("status=%d calls=%d body=%s", recorder.Code, calls.Load(), recorder.Body) + } + }) + } + } + } + } +} + +func TestGatewayOpenRouterProviderKeepsToolPreparation(t *testing.T) { + const originalPattern = `^a\0(?=b)b$` + const normalizedPattern = `^a\x00b$` + for _, model := range []string{"deepseek/deepseek-chat", "moonshotai/kimi-k2", "anthropic/claude-sonnet-4.6"} { + for _, client := range []protocol.Protocol{protocol.OpenAICompletions, protocol.OpenAIResponses, protocol.Anthropic, protocol.Gemini} { + for _, preferences := range []bool{false, true} { + for _, stream := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/provider=%t/stream=%t", model, client, preferences, stream), func(t *testing.T) { + var calls atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + return + } + if body["model"] != model || r.Header.Get("Authorization") != "Bearer sk-upstream" { + t.Errorf("upstream model or credential changed: %#v", body) + } + tool := body["tools"].([]any)[0].(map[string]any) + if client == protocol.OpenAICompletions { + tool = tool["function"].(map[string]any) + } + properties := tool["parameters"].(map[string]any)["properties"].(map[string]any) + value := properties["value"].(map[string]any) + wantPattern, wantFormat := originalPattern, "password" + converted := client == protocol.Anthropic || client == protocol.Gemini + if converted && model != "anthropic/claude-sonnet-4.6" { + wantPattern = normalizedPattern + } + if converted && model == "moonshotai/kimi-k2" { + wantFormat = "" + } + if value["pattern"] != wantPattern { + t.Errorf("pattern = %q, want %q", value["pattern"], wantPattern) + } + gotFormat, _ := value["format"].(string) + if gotFormat != wantFormat { + t.Errorf("format = %q, want %q", gotFormat, wantFormat) + } + if properties["email"].(map[string]any)["format"] != "email" { + t.Error("allowed format lost") + } + provider, present := body["provider"] + if present != preferences || present && !reflect.DeepEqual(provider, map[string]any{"only": []any{"modal/fp4"}}) { + t.Errorf("provider = %#v", provider) + } + response := `{"id":"resp_1","object":"response","model":"test","status":"completed","output":[]}` + streamResponse := "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":" + response + "}\n\n" + path := "/v1/responses" + if client == protocol.OpenAICompletions { + path = "/v1/chat/completions" + response = `{"id":"chatcmpl-1","object":"chat.completion","model":"test","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}` + streamResponse = "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n" + } + if r.URL.Path != path { + t.Errorf("path = %s, want %s", r.URL.Path, path) + } + w.Header().Set("Content-Type", "application/json") + if stream { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, streamResponse) + } else { + _, _ = io.WriteString(w, response) + } + })) + defer upstream.Close() + schema := map[string]any{"type": "object", "properties": map[string]any{"value": map[string]any{"type": "string", "pattern": originalPattern, "format": "password"}, "email": map[string]any{"type": "string", "format": "email"}}} + body := map[string]any{"model": "public-model", "stream": stream} + path := "/v1/responses" + switch client { + case protocol.OpenAICompletions: + path = "/v1/chat/completions" + body["messages"] = []any{map[string]any{"role": "user", "content": "hello"}} + body["tools"] = []any{map[string]any{"type": "function", "function": map[string]any{"name": "lookup", "parameters": schema}}} + case protocol.OpenAIResponses: + body["input"] = "hello" + body["tools"] = []any{map[string]any{"type": "function", "name": "lookup", "parameters": schema}} + case protocol.Anthropic: + path = "/v1/messages" + body["max_tokens"] = 16 + body["messages"] = []any{map[string]any{"role": "user", "content": "hello"}} + body["tools"] = []any{map[string]any{"name": "lookup", "input_schema": schema}} + case protocol.Gemini: + path = "/v1beta/models/public-model:generateContent" + if stream { + path = "/v1beta/models/public-model:streamGenerateContent?alt=sse" + } + body["contents"] = []any{map[string]any{"role": "user", "parts": []any{map[string]any{"text": "hello"}}}} + body["tools"] = []any{map[string]any{"functionDeclarations": []any{map[string]any{"name": "lookup", "parametersJsonSchema": schema}}}} + } + if preferences { + body["provider"] = map[string]any{"only": []any{"modal/fp4"}} + } + params, _ := json.Marshal(map[string]string{"base_url": upstream.URL}) + engine, _ := newDialectGatewayEngine(t, client, "public-model", dialect.NewSet(dialect.NewOpenAI(), dialect.NewOpenAIResponses(), dialect.NewAnthropic(), dialect.NewGemini()), dialectGatewayGroup{ + id: 1, name: "test", channelID: channel.OpenRouter, params: params, apiKeys: []string{"sk-upstream"}, models: []state.ModelConfig{{ID: model, Alias: "public-model"}}, + }) + encoded, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, path, strings.NewReader(string(encoded))) + request.Header.Set("Authorization", "Bearer gl-client") + recorder := httptest.NewRecorder() + engine.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK || calls.Load() != 1 { + t.Fatalf("status=%d calls=%d body=%s", recorder.Code, calls.Load(), recorder.Body) + } + }) + } + } + } + } +} diff --git a/internal/gateway/openrouter_provider_test.go b/internal/gateway/openrouter_provider_test.go new file mode 100644 index 0000000000..92d13911f9 --- /dev/null +++ b/internal/gateway/openrouter_provider_test.go @@ -0,0 +1,122 @@ +package gateway + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync/atomic" + "testing" + + "gpt-load/internal/channel" + "gpt-load/internal/dialect" + bifrostexecutor "gpt-load/internal/execution/bifrost" + "gpt-load/internal/state" +) + +func TestGatewayOpenRouterProviderPreferences(t *testing.T) { + const preferences = `{"only":["modal/fp4"],"allow_fallbacks":false,"order":["modal/fp4"],"require_parameters":true}` + for _, test := range []struct { + name string + channel channel.ID + provider string + preserve bool + }{ + {"OpenRouter preferences", channel.OpenRouter, preferences, true}, + {"OpenRouter scalar control", channel.OpenRouter, `"anthropic"`, false}, + {"OpenRouter null control", channel.OpenRouter, `null`, false}, + {"OpenRouter array control", channel.OpenRouter, `["modal/fp4"]`, false}, + {"OpenRouter empty preferences", channel.OpenRouter, `{}`, true}, + {"OpenAI control", channel.OpenAI, preferences, false}, + {"DeepSeek control", channel.DeepSeek, preferences, false}, + } { + for _, responses := range []bool{false, true} { + for _, stream := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/responses=%t/stream=%t", test.name, responses, stream), func(t *testing.T) { + path := "/v1/chat/completions" + input := `"messages":[{"role":"user","content":"hello"}]` + response := `{"id":"chatcmpl-1","object":"chat.completion","model":"upstream-model","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}` + streamResponse := "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"upstream-model\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n" + if responses { + path = "/v1/responses" + input = `"input":"hello"` + response = `{"id":"resp_1","object":"response","model":"upstream-model","status":"completed","output":[]}` + streamResponse = "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":" + response + "}\n\n" + } + var calls atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.Method != http.MethodPost || !strings.HasSuffix(r.URL.Path, strings.TrimPrefix(path, "/v1")) || r.Header.Get("Authorization") != "Bearer sk-upstream" { + t.Errorf("unexpected upstream route or credential: %s %s", r.Method, r.URL.Path) + } + var body map[string]json.RawMessage + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + return + } + if string(body["model"]) != `"upstream-model"` || string(body["stream"]) != fmt.Sprint(stream) { + t.Errorf("model or stream changed: %s", body) + } + if test.preserve { + var got, want any + if err := json.Unmarshal(body["provider"], &got); err != nil { + t.Errorf("provider preferences missing: %s", body) + } + if err := json.Unmarshal([]byte(test.provider), &want); err != nil { + t.Error(err) + return + } + if !reflect.DeepEqual(got, want) { + t.Errorf("provider preferences = %#v, want %#v", got, want) + } + } else if _, exists := body["provider"]; exists { + t.Errorf("provider control reached upstream: %s", body["provider"]) + } + for _, key := range []string{"Provider", "fallback", "fallbacks", "api_key", "Authorization", "proxy-authorization", "x-api-key", "x-goog-api-key"} { + if _, exists := body[key]; exists { + t.Errorf("control %q reached upstream", key) + } + } + w.Header().Set("Content-Type", "application/json") + if stream { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, streamResponse) + } else { + _, _ = io.WriteString(w, response) + } + })) + defer upstream.Close() + runtime, err := bifrostexecutor.NewRuntime(context.Background(), channel.NewRegistry()) + if err != nil { + t.Fatal(err) + } + defer runtime.Shutdown() + engine, handler, manager, _ := newRequestLogHandlerTestRuntime(t, NewExecutionForwarder(runtime), &recordingAccessKeyRPMLimiter{}, &recordingRequestLogSink{}, "sk-upstream") + handler.dialects = dialect.NewSet(dialect.NewOpenAI(), dialect.NewOpenAIResponses()) + params, _ := json.Marshal(map[string]string{"base_url": upstream.URL}) + _, err = manager.Publish(state.CompileInput{ + ChannelRegistry: channel.NewRegistry(), + Groups: []state.GroupConfig{{ConnectionType: "api_key", ID: 1, Name: "test", ChannelID: test.channel, Params: params, Models: []state.ModelConfig{{ID: "upstream-model", Alias: "public"}}, Enabled: true}}, + Credentials: []state.CredentialConfig{testCredentialConfig(1, 1)}, + AccessKeys: []state.AccessKeyConfig{{ID: 1, Name: "client", KeyHash: handler.encryption.Hash("gl-client"), Status: state.AccessKeyStatusActive}}, + }) + if err != nil { + t.Fatal(err) + } + body := fmt.Sprintf(`{"model":"public",%s,"stream":%t,"provider":%s,"Provider":{"only":["other"]},"fallback":"other","fallbacks":["other"],"api_key":"client-key","Authorization":"client-key","proxy-authorization":"client-key","x-api-key":"client-key","x-goog-api-key":"client-key"}`, input, stream, test.provider) + request := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)) + request.Header.Set("Authorization", "Bearer gl-client") + recorder := httptest.NewRecorder() + engine.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK || calls.Load() != 1 { + t.Fatalf("status=%d calls=%d body=%s", recorder.Code, calls.Load(), recorder.Body) + } + }) + } + } + } +}